* Copyright (c) 1996, 1997, 1998, 1999, 2007, 2009 The NetBSD Foundation, Inc.

/* $NetBSD: ccd.c,v 1.192 2025/01/08 08:24:07 andvar Exp $ */

/*-
* Copyright (c) 1996, 1997, 1998, 1999, 2007, 2009 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
* by Jason R. Thorpe, and by Andrew Doran.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/

/*
* Copyright (c) 1988 University of Utah.
* Copyright (c) 1990, 1993
* The Regents of the University of California. All rights reserved.
*
* This code is derived from software contributed to Berkeley by
* the Systems Programming Group of the University of Utah Computer
* Science Department.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* from: Utah $Hdr: cd.c 1.6 90/11/28$
*
* @(#)cd.c 8.2 (Berkeley) 11/16/93
*/

/*
* "Concatenated" disk driver.
*
* Notes on concurrency:
*
* => sc_dvlock serializes access to the device nodes, excluding block I/O.
*
* => sc_iolock serializes access to (sc_flags & CCDF_INITED), disk stats,
* sc_stop, sc_bufq and b_resid from master buffers.
*
* => a combination of CCDF_INITED, sc_inflight, and sc_iolock is used to
* serialize I/O and configuration changes.
*
* => the in-core disk label does not change while the device is open.
*
* On memory consumption: ccd fans out I/O requests and so needs to
* allocate memory. If the system is desperately low on memory, we
* single thread I/O.
*/

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: ccd.c,v 1.192 2025/01/08 08:24:07 andvar Exp $");

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/proc.h>
#include <sys/errno.h>
#include <sys/buf.h>
#include <sys/kmem.h>
#include <sys/pool.h>
#include <sys/module.h>
#include <sys/namei.h>
#include <sys/stat.h>
#include <sys/ioctl.h>
#include <sys/disklabel.h>
#include <sys/device.h>
#include <sys/disk.h>
#include <sys/syslog.h>
#include <sys/fcntl.h>
#include <sys/vnode.h>
#include <sys/conf.h>
#include <sys/mutex.h>
#include <sys/queue.h>
#include <sys/kauth.h>
#include <sys/kthread.h>
#include <sys/bufq.h>
#include <sys/sysctl.h>
#include <sys/compat_stub.h>

#include <uvm/uvm_extern.h>

#include <dev/ccdvar.h>
#include <dev/dkvar.h>

#include <miscfs/specfs/specdev.h> /* for v_rdev */

#include "ioconf.h"

#if defined(CCDDEBUG) && !defined(DEBUG)
#define DEBUG
#endif

#ifdef DEBUG
#define CCDB_FOLLOW 0x01
#define CCDB_INIT 0x02
#define CCDB_IO 0x04
#define CCDB_LABEL 0x08
#define CCDB_VNODE 0x10
int ccddebug = 0x00;
#endif

#define ccdunit(x) DISKUNIT(x)

struct ccdbuf {
struct buf cb_buf; /* new I/O buf */
struct buf *cb_obp; /* ptr. to original I/O buf */
struct ccd_softc *cb_sc; /* pointer to ccd softc */
int cb_comp; /* target component */
SIMPLEQ_ENTRY(ccdbuf) cb_q; /* fifo of component buffers */
};

/* component buffer pool */
static pool_cache_t ccd_cache;

#define CCD_GETBUF(wait) pool_cache_get(ccd_cache, wait)
#define CCD_PUTBUF(cbp) pool_cache_put(ccd_cache, cbp)

#define CCDLABELDEV(dev) \
(MAKEDISKDEV(major((dev)), ccdunit((dev)), RAW_PART))

/* called by main() at boot time */
void ccddetach(void);

/* called by biodone() at interrupt time */
static void ccdiodone(struct buf *);

static void ccdinterleave(struct ccd_softc *);
static int ccdinit(struct ccd_softc *, char **, struct vnode **,
struct lwp *);
static struct ccdbuf *ccdbuffer(struct ccd_softc *, struct buf *,
daddr_t, void *, long, int);
static void ccdgetdefaultlabel(struct ccd_softc *, struct disklabel *);
static void ccdgetdisklabel(dev_t);
static void ccdmakedisklabel(struct ccd_softc *);
static int ccdstart(struct ccd_softc *, struct buf *, int);
static void ccdthread(void *);

static dev_type_open(ccdopen);
static dev_type_close(ccdclose);
static dev_type_read(ccdread);
static dev_type_write(ccdwrite);
static dev_type_ioctl(ccdioctl);
static dev_type_strategy(ccdstrategy);
static dev_type_size(ccdsize);

const struct bdevsw ccd_bdevsw = {
.d_open = ccdopen,
.d_close = ccdclose,
.d_strategy = ccdstrategy,
.d_ioctl = ccdioctl,
.d_dump = nodump,
.d_psize = ccdsize,
.d_discard = nodiscard,
.d_flag = D_DISK | D_MPSAFE
};

const struct cdevsw ccd_cdevsw = {
.d_open = ccdopen,
.d_close = ccdclose,
.d_read = ccdread,
.d_write = ccdwrite,
.d_ioctl = ccdioctl,
.d_stop = nostop,
.d_tty = notty,
.d_poll = nopoll,
.d_mmap = nommap,
.d_kqfilter = nokqfilter,
.d_discard = nodiscard,
.d_flag = D_DISK | D_MPSAFE
};

static const struct dkdriver ccddkdriver = {
.d_strategy = ccdstrategy,
.d_minphys = minphys
};

#ifdef DEBUG
static void printiinfo(struct ccdiinfo *);
#endif

static LIST_HEAD(, ccd_softc) ccds = LIST_HEAD_INITIALIZER(ccds);
static kmutex_t ccd_lock;

SYSCTL_SETUP_PROTO(sysctl_kern_ccd_setup);

static struct ccd_softc *
ccdcreate(int unit) {
struct ccd_softc *sc = kmem_zalloc(sizeof(*sc), KM_SLEEP);

/* Initialize per-softc structures. */
snprintf(sc->sc_xname, sizeof(sc->sc_xname), "ccd%d", unit);
sc->sc_unit = unit;
mutex_init(&sc->sc_dvlock, MUTEX_DEFAULT, IPL_NONE);
sc->sc_iolock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
cv_init(&sc->sc_stop, "ccdstop");
cv_init(&sc->sc_push, "ccdthr");
disk_init(&sc->sc_dkdev, sc->sc_xname, &ccddkdriver);
return sc;
}

static void
ccddestroy(struct ccd_softc *sc) {
mutex_obj_free(sc->sc_iolock);
mutex_exit(&sc->sc_dvlock);
mutex_destroy(&sc->sc_dvlock);
cv_destroy(&sc->sc_stop);
cv_destroy(&sc->sc_push);
disk_destroy(&sc->sc_dkdev);
kmem_free(sc, sizeof(*sc));
}

static struct ccd_softc *
ccdget(int unit, int make) {
struct ccd_softc *sc;
if (unit < 0) {
#ifdef DIAGNOSTIC
panic("%s: unit %d!", __func__, unit);
#endif
return NULL;
}
mutex_enter(&ccd_lock);
LIST_FOREACH(sc, &ccds, sc_link) {
if (sc->sc_unit == unit) {
mutex_exit(&ccd_lock);
return sc;
}
}
mutex_exit(&ccd_lock);
if (!make)
return NULL;
if ((sc = ccdcreate(unit)) == NULL)
return NULL;
mutex_enter(&ccd_lock);
LIST_INSERT_HEAD(&ccds, sc, sc_link);
mutex_exit(&ccd_lock);
return sc;
}

static void
ccdput(struct ccd_softc *sc) {
mutex_enter(&ccd_lock);
LIST_REMOVE(sc, sc_link);
mutex_exit(&ccd_lock);
ccddestroy(sc);
}

/*
* Called by main() during pseudo-device attachment. All we need
* to do is allocate enough space for devices to be configured later.
*/
void
ccdattach(int num)
{
mutex_init(&ccd_lock, MUTEX_DEFAULT, IPL_NONE);

/* Initialize the component buffer pool. */
ccd_cache = pool_cache_init(sizeof(struct ccdbuf), 0,
0, 0, "ccdbuf", NULL, IPL_BIO, NULL, NULL, NULL);
}

void
ccddetach(void)
{
pool_cache_destroy(ccd_cache);
mutex_destroy(&ccd_lock);
}

static int
ccdinit(struct ccd_softc *cs, char **cpaths, struct vnode **vpp,
struct lwp *l)
{
struct ccdcinfo *ci = NULL;
int ix;
struct ccdgeom *ccg = &cs->sc_geom;
char *tmppath;
int error, path_alloced;
uint64_t psize, minsize;
unsigned secsize, maxsecsize;
struct disk_geom *dg;

#ifdef DEBUG
if (ccddebug & (CCDB_FOLLOW|CCDB_INIT))
printf("%s: ccdinit\n", cs->sc_xname);
#endif

/* Allocate space for the component info. */
cs->sc_cinfo = kmem_alloc(cs->sc_nccdisks * sizeof(*cs->sc_cinfo),
KM_SLEEP);
tmppath = kmem_alloc(MAXPATHLEN, KM_SLEEP);

cs->sc_size = 0;

/*
* Verify that each component piece exists and record
* relevant information about it.
*/
maxsecsize = 0;
minsize = 0;
for (ix = 0, path_alloced = 0; ix < cs->sc_nccdisks; ix++) {
ci = &cs->sc_cinfo[ix];
ci->ci_vp = vpp[ix];

/*
* Copy in the pathname of the component.
*/
memset(tmppath, 0, MAXPATHLEN); /* sanity */
error = copyinstr(cpaths[ix], tmppath,
MAXPATHLEN, &ci->ci_pathlen);
if (ci->ci_pathlen == 0)
error = EINVAL;
if (error) {
#ifdef DEBUG
if (ccddebug & (CCDB_FOLLOW|CCDB_INIT))
printf("%s: can't copy path, error = %d\n",
cs->sc_xname, error);
#endif
goto out;
}
ci->ci_path = kmem_alloc(ci->ci_pathlen, KM_SLEEP);
memcpy(ci->ci_path, tmppath, ci->ci_pathlen);
path_alloced++;

/*
* XXX: Cache the component's dev_t.
*/
ci->ci_dev = vpp[ix]->v_rdev;

/*
* Get partition information for the component.
*/
error = getdisksize(vpp[ix], &psize, &secsize);
if (error) {
#ifdef DEBUG
if (ccddebug & (CCDB_FOLLOW|CCDB_INIT))
printf("%s: %s: disksize failed, error = %d\n",
cs->sc_xname, ci->ci_path, error);
#endif
goto out;
}

/*
* Calculate the size, truncating to an interleave
* boundary if necessary.
*/
maxsecsize = secsize > maxsecsize ? secsize : maxsecsize;
if (cs->sc_ileave > 1)
psize -= psize % cs->sc_ileave;

if (psize == 0) {
#ifdef DEBUG
if (ccddebug & (CCDB_FOLLOW|CCDB_INIT))
printf("%s: %s: size == 0\n",
cs->sc_xname, ci->ci_path);
#endif
error = ENODEV;
goto out;
}

if (minsize == 0 || psize < minsize)
minsize = psize;
ci->ci_size = psize;
cs->sc_size += psize;
}

/*
* Don't allow the interleave to be smaller than
* the biggest component sector.
*/
if ((cs->sc_ileave > 0) &&
(cs->sc_ileave < (maxsecsize / DEV_BSIZE))) {
#ifdef DEBUG
if (ccddebug & (CCDB_FOLLOW|CCDB_INIT))
printf("%s: interleave must be at least %d\n",
cs->sc_xname, (maxsecsize / DEV_BSIZE));
#endif
error = EINVAL;
goto out;
}

/*
* If uniform interleave is desired set all sizes to that of
* the smallest component.
*/
if (cs->sc_flags & CCDF_UNIFORM) {
for (ci = cs->sc_cinfo;
ci < &cs->sc_cinfo[cs->sc_nccdisks]; ci++)
ci->ci_size = minsize;

cs->sc_size = cs->sc_nccdisks * minsize;
}

/*
* Construct the interleave table.
*/
ccdinterleave(cs);

/*
* Create pseudo-geometry based on 1MB cylinders. It's
* pretty close.
*/
ccg->ccg_secsize = DEV_BSIZE;
ccg->ccg_ntracks = 1;
ccg->ccg_nsectors = 1024 * (1024 / ccg->ccg_secsize);
ccg->ccg_ncylinders = cs->sc_size / ccg->ccg_nsectors;

dg = &cs->sc_dkdev.dk_geom;
memset(dg, 0, sizeof(*dg));
dg->dg_secperunit = cs->sc_size;
dg->dg_secsize = ccg->ccg_secsize;
dg->dg_nsectors = ccg->ccg_nsectors;
dg->dg_ntracks = ccg->ccg_ntracks;
dg->dg_ncylinders = ccg->ccg_ncylinders;

if (cs->sc_ileave > 0)
aprint_normal("%s: Interleaving %d component%s "
"(%d block interleave)\n", cs->sc_xname,
cs->sc_nccdisks, (cs->sc_nccdisks != 0 ? "s" : ""),
cs->sc_ileave);
else
aprint_normal("%s: Concatenating %d component%s\n",
cs->sc_xname,
cs->sc_nccdisks, (cs->sc_nccdisks != 0 ? "s" : ""));
for (ix = 0; ix < cs->sc_nccdisks; ix++) {
ci = &cs->sc_cinfo[ix];
aprint_normal("%s: %s (%ju blocks)\n", cs->sc_xname,
ci->ci_path, (uintmax_t)ci->ci_size);
}
aprint_normal("%s: total %ju blocks\n", cs->sc_xname, cs->sc_size);

/*
* Create thread to handle deferred I/O.
*/
cs->sc_zap = false;
error = kthread_create(PRI_BIO, KTHREAD_MPSAFE, NULL, ccdthread,
cs, &cs->sc_thread, "%s", cs->sc_xname);
if (error) {
printf("ccdinit: can't create thread: %d\n", error);
goto out;
}

/*
* Only now that everything is set up can we enable the device.
*/
mutex_enter(cs->sc_iolock);
cs->sc_flags |= CCDF_INITED;
mutex_exit(cs->sc_iolock);
kmem_free(tmppath, MAXPATHLEN);
return (0);

out:
for (ix = 0; ix < path_alloced; ix++) {
kmem_free(cs->sc_cinfo[ix].ci_path,
cs->sc_cinfo[ix].ci_pathlen);
}
kmem_free(cs->sc_cinfo, cs->sc_nccdisks * sizeof(struct ccdcinfo));
kmem_free(tmppath, MAXPATHLEN);
return (error);
}

static void
ccdinterleave(struct ccd_softc *cs)
{
struct ccdcinfo *ci, *smallci;
struct ccdiinfo *ii;
daddr_t bn, lbn;
int ix;
u_long size;

#ifdef DEBUG
if (ccddebug & CCDB_INIT)
printf("ccdinterleave(%p): ileave %d\n", cs, cs->sc_ileave);
#endif
/*
* Allocate an interleave table.
* Chances are this is too big, but we don't care.
*/
size = (cs->sc_nccdisks + 1) * sizeof(struct ccdiinfo);
cs->sc_itable = kmem_zalloc(size, KM_SLEEP);

/*
* Trivial case: no interleave (actually interleave of disk size).
* Each table entry represents a single component in its entirety.
*/
if (cs->sc_ileave == 0) {
bn = 0;
ii = cs->sc_itable;

for (ix = 0; ix < cs->sc_nccdisks; ix++) {
/* Allocate space for ii_index. */
ii->ii_indexsz = sizeof(int);
ii->ii_index = kmem_alloc(ii->ii_indexsz, KM_SLEEP);
ii->ii_ndisk = 1;
ii->ii_startblk = bn;
ii->ii_startoff = 0;
ii->ii_index[0] = ix;
bn += cs->sc_cinfo[ix].ci_size;
ii++;
}
ii->ii_ndisk = 0;
#ifdef DEBUG
if (ccddebug & CCDB_INIT)
printiinfo(cs->sc_itable);
#endif
return;
}

/*
* The following isn't fast or pretty; it doesn't have to be.
*/
size = 0;
bn = lbn = 0;
for (ii = cs->sc_itable; ; ii++) {
/* Allocate space for ii_index. */
ii->ii_indexsz = sizeof(int) * cs->sc_nccdisks;
ii->ii_index = kmem_alloc(ii->ii_indexsz, KM_SLEEP);

/*
* Locate the smallest of the remaining components
*/
smallci = NULL;
for (ci = cs->sc_cinfo;
ci < &cs->sc_cinfo[cs->sc_nccdisks]; ci++)
if (ci->ci_size > size &&
(smallci == NULL ||
ci->ci_size < smallci->ci_size))
smallci = ci;

/*
* Nobody left, all done
*/
if (smallci == NULL) {
ii->ii_ndisk = 0;
break;
}

/*
* Record starting logical block and component offset
*/
ii->ii_startblk = bn / cs->sc_ileave;
ii->ii_startoff = lbn;

/*
* Determine how many disks take part in this interleave
* and record their indices.
*/
ix = 0;
for (ci = cs->sc_cinfo;
ci < &cs->sc_cinfo[cs->sc_nccdisks]; ci++)
if (ci->ci_size >= smallci->ci_size)
ii->ii_index[ix++] = ci - cs->sc_cinfo;
ii->ii_ndisk = ix;
bn += ix * (smallci->ci_size - size);
lbn = smallci->ci_size / cs->sc_ileave;
size = smallci->ci_size;
}
#ifdef DEBUG
if (ccddebug & CCDB_INIT)
printiinfo(cs->sc_itable);
#endif
}

/* ARGSUSED */
static int
ccdopen(dev_t dev, int flags, int fmt, struct lwp *l)
{
int unit = ccdunit(dev);
struct ccd_softc *cs;
struct disklabel *lp;
int error = 0, part, pmask;

#ifdef DEBUG
if (ccddebug & CCDB_FOLLOW)
printf("ccdopen(0x%"PRIx64", 0x%x)\n", dev, flags);
#endif
if ((cs = ccdget(unit, 1)) == NULL)
return ENXIO;

mutex_enter(&cs->sc_dvlock);

lp = cs->sc_dkdev.dk_label;

part = DISKPART(dev);
pmask = (1 << part);

/*
* If we're initialized, check to see if there are any other
* open partitions. If not, then it's safe to update
* the in-core disklabel. Only read the disklabel if it is
* not already valid.
*/
if ((cs->sc_flags & (CCDF_INITED|CCDF_VLABEL)) == CCDF_INITED &&
cs->sc_dkdev.dk_openmask == 0)
ccdgetdisklabel(dev);

/* Check that the partition exists. */
if (part != RAW_PART) {
if (((cs->sc_flags & CCDF_INITED) == 0) ||
((part >= lp->d_npartitions) ||
(lp->d_partitions[part].p_fstype == FS_UNUSED))) {
error = ENXIO;
goto done;
}
}

/* Prevent our unit from being unconfigured while open. */
switch (fmt) {
case S_IFCHR:
cs->sc_dkdev.dk_copenmask |= pmask;
break;

case S_IFBLK:
cs->sc_dkdev.dk_bopenmask |= pmask;
break;
}
cs->sc_dkdev.dk_openmask =
cs->sc_dkdev.dk_copenmask | cs->sc_dkdev.dk_bopenmask;

done:
mutex_exit(&cs->sc_dvlock);
return (error);
}

/* ARGSUSED */
static int
ccdclose(dev_t dev, int flags, int fmt, struct lwp *l)
{
int unit = ccdunit(dev);
struct ccd_softc *cs;
int part;

#ifdef DEBUG
if (ccddebug & CCDB_FOLLOW)
printf("ccdclose(0x%"PRIx64", 0x%x)\n", dev, flags);
#endif

if ((cs = ccdget(unit, 0)) == NULL)
return ENXIO;

mutex_enter(&cs->sc_dvlock);

part = DISKPART(dev);

/* ...that much closer to allowing unconfiguration... */
switch (fmt) {
case S_IFCHR:
cs->sc_dkdev.dk_copenmask &= ~(1 << part);
break;

case S_IFBLK:
cs->sc_dkdev.dk_bopenmask &= ~(1 << part);
break;
}
cs->sc_dkdev.dk_openmask =
cs->sc_dkdev.dk_copenmask | cs->sc_dkdev.dk_bopenmask;

if (cs->sc_dkdev.dk_openmask == 0) {
if ((cs->sc_flags & CCDF_KLABEL) == 0)
cs->sc_flags &= ~CCDF_VLABEL;
}

mutex_exit(&cs->sc_dvlock);
return (0);
}

static void
ccdthread(void *cookie)
{
int error;
struct ccd_softc *cs;
struct buf *bp;

cs = cookie;

#ifdef DEBUG
if (ccddebug & CCDB_FOLLOW)
printf("ccdthread: hello\n");
#endif

mutex_enter(cs->sc_iolock);
while (__predict_true(!cs->sc_zap)) {
bp = bufq_get(cs->sc_bufq);
if (bp == NULL) {
/* Nothing to do. */
cv_wait(&cs->sc_push, cs->sc_iolock);
continue;
}
#ifdef DEBUG
if (ccddebug & CCDB_FOLLOW)
printf("ccdthread: dispatching I/O\n");
#endif
error = ccdstart(cs, bp, PR_WAITOK);
KASSERT(error == 0);
mutex_enter(cs->sc_iolock);
}
cs->sc_thread = NULL;
mutex_exit(cs->sc_iolock);
#ifdef DEBUG
if (ccddebug & CCDB_FOLLOW)
printf("ccdthread: goodbye\n");
#endif
kthread_exit(0);
}

static void
ccdstrategy(struct buf *bp)
{
int unit = ccdunit(bp->b_dev);
struct ccd_softc *cs;
if ((cs = ccdget(unit, 0)) == NULL)
return;

/* Must be open or reading label. */
KASSERT(cs->sc_dkdev.dk_openmask != 0 ||
(cs->sc_flags & CCDF_RLABEL) != 0);

mutex_enter(cs->sc_iolock);
/* Synchronize with device init/uninit. */
if (__predict_false((cs->sc_flags & CCDF_INITED) == 0)) {
mutex_exit(cs->sc_iolock);
#ifdef DEBUG
if (ccddebug & CCDB_FOLLOW)
printf("ccdstrategy: unit %d: not inited\n", unit);
#endif
bp->b_error = ENXIO;
bp->b_resid = bp->b_bcount;
biodone(bp);
return;
}

if (ccdstart(cs, bp, PR_NOWAIT) != 0) {
/* Defer to thread if system is low on memory. */
bufq_put(cs->sc_bufq, bp);
cv_broadcast(&cs->sc_push);
mutex_exit(cs->sc_iolock);
}
}

static int
ccdstart(struct ccd_softc *cs, struct buf *bp, int wait)
{
daddr_t blkno;
int wlabel;
struct disklabel *lp;
long bcount, rcount;
struct ccdbuf *cbp;
char *addr;
daddr_t bn;
vnode_t *vp;
SIMPLEQ_HEAD(, ccdbuf) cbufq;

KASSERT(mutex_owned(cs->sc_iolock));
KASSERT(bp != NULL);

disk_busy(&cs->sc_dkdev);

#ifdef DEBUG
if (ccddebug & CCDB_FOLLOW)
printf("ccdstart(%s, %p)\n", cs->sc_xname, bp);
#endif

/* If it's a nil transfer, wake up the top half now. */
if (bp->b_bcount == 0)
goto done;

lp = cs->sc_dkdev.dk_label;

/*
* Do bounds checking and adjust transfer. If there's an
* error, the bounds check will flag that for us. Convert
* the partition relative block number to an absolute.
*/
blkno = bp->b_blkno;
wlabel = cs->sc_flags & (CCDF_WLABEL|CCDF_LABELLING);
if (DISKPART(bp->b_dev) != RAW_PART) {
if (bounds_check_with_label(&cs->sc_dkdev, bp, wlabel) <= 0)
goto done;
blkno += lp->d_partitions[DISKPART(bp->b_dev)].p_offset;
}
mutex_exit(cs->sc_iolock);
bp->b_rawblkno = blkno;

/* Allocate the component buffers. */
SIMPLEQ_INIT(&cbufq);
bp->b_resid = bp->b_bcount;
bn = bp->b_rawblkno;
addr = bp->b_data;
for (bcount = bp->b_bcount; bcount > 0; bcount -= rcount) {
cbp = ccdbuffer(cs, bp, bn, addr, bcount, wait);
KASSERT(cbp != NULL || wait == PR_NOWAIT);
if (cbp == NULL) {
while ((cbp = SIMPLEQ_FIRST(&cbufq)) != NULL) {
SIMPLEQ_REMOVE_HEAD(&cbufq, cb_q);
CCD_PUTBUF(cbp);
}
mutex_enter(cs->sc_iolock);
disk_unbusy(&cs->sc_dkdev, 0, 0);
return ENOMEM;
}
SIMPLEQ_INSERT_TAIL(&cbufq, cbp, cb_q);
rcount = cbp->cb_buf.b_bcount;
bn += btodb(rcount);
addr += rcount;
}

/* All buffers set up, now fire off the requests. */
while ((cbp = SIMPLEQ_FIRST(&cbufq)) != NULL) {
SIMPLEQ_REMOVE_HEAD(&cbufq, cb_q);
vp = cbp->cb_buf.b_vp;
if ((cbp->cb_buf.b_flags & B_READ) == 0) {
mutex_enter(vp->v_interlock);
vp->v_numoutput++;
mutex_exit(vp->v_interlock);
}
(void)VOP_STRATEGY(vp, &cbp->cb_buf);
}
return 0;

done:
disk_unbusy(&cs->sc_dkdev, 0, 0);
cv_broadcast(&cs->sc_stop);
cv_broadcast(&cs->sc_push);
mutex_exit(cs->sc_iolock);
bp->b_resid = bp->b_bcount;
biodone(bp);
return 0;
}

/*
* Build a component buffer header.
*/
static struct ccdbuf *
ccdbuffer(struct ccd_softc *cs, struct buf *bp, daddr_t bn, void *addr,
long bcount, int wait)
{
struct ccdcinfo *ci;
struct ccdbuf *cbp;
daddr_t cbn, cboff;
u_int64_t cbc;
int ccdisk;

#ifdef DEBUG
if (ccddebug & CCDB_IO)
printf("ccdbuffer(%p, %p, %" PRId64 ", %p, %ld)\n",
cs, bp, bn, addr, bcount);
#endif
/*
* Determine which component bn falls in.
*/
cbn = bn;
cboff = 0;

/*
* Serially concatenated
*/
if (cs->sc_ileave == 0) {
daddr_t sblk;

sblk = 0;
for (ccdisk = 0, ci = &cs->sc_cinfo[ccdisk];
cbn >= sblk + ci->ci_size;
ccdisk++, ci = &cs->sc_cinfo[ccdisk])
sblk += ci->ci_size;
cbn -= sblk;
}
/*
* Interleaved
*/
else {
struct ccdiinfo *ii;
int off;

cboff = cbn % cs->sc_ileave;
cbn /= cs->sc_ileave;
for (ii = cs->sc_itable; ii->ii_ndisk; ii++)
if (ii->ii_startblk > cbn)
break;
ii--;
off = cbn - ii->ii_startblk;
if (ii->ii_ndisk == 1) {
ccdisk = ii->ii_index[0];
cbn = ii->ii_startoff + off;
} else {
ccdisk = ii->ii_index[off % ii->ii_ndisk];
cbn = ii->ii_startoff + off / ii->ii_ndisk;
}
cbn *= cs->sc_ileave;
ci = &cs->sc_cinfo[ccdisk];
}

/*
* Fill in the component buf structure.
*/
cbp = CCD_GETBUF(wait);
if (cbp == NULL)
return NULL;
buf_init(&cbp->cb_buf);
cbp->cb_buf.b_flags = bp->b_flags;
cbp->cb_buf.b_oflags = bp->b_oflags;
cbp->cb_buf.b_cflags = bp->b_cflags;
cbp->cb_buf.b_iodone = ccdiodone;
cbp->cb_buf.b_proc = bp->b_proc;
cbp->cb_buf.b_dev = ci->ci_dev;
cbp->cb_buf.b_blkno = cbn + cboff;
cbp->cb_buf.b_data = addr;
cbp->cb_buf.b_vp = ci->ci_vp;
cbp->cb_buf.b_objlock = ci->ci_vp->v_interlock;
if (cs->sc_ileave == 0)
cbc = dbtob((u_int64_t)(ci->ci_size - cbn));
else
cbc = dbtob((u_int64_t)(cs->sc_ileave - cboff));
cbp->cb_buf.b_bcount = cbc < bcount ? cbc : bcount;

/*
* context for ccdiodone
*/
cbp->cb_obp = bp;
cbp->cb_sc = cs;
cbp->cb_comp = ccdisk;

BIO_COPYPRIO(&cbp->cb_buf, bp);

#ifdef DEBUG
if (ccddebug & CCDB_IO)
printf(" dev 0x%"PRIx64"(u%lu): cbp %p bn %" PRId64 " addr %p"
" bcnt %d\n",
ci->ci_dev, (unsigned long) (ci-cs->sc_cinfo), cbp,
cbp->cb_buf.b_blkno, cbp->cb_buf.b_data,
cbp->cb_buf.b_bcount);
#endif

return (cbp);
}

/*
* Called at interrupt time.
* Mark the component as done and if all components are done,
* take a ccd interrupt.
*/
static void
ccdiodone(struct buf *vbp)
{
struct ccdbuf *cbp = (struct ccdbuf *) vbp;
struct buf *bp = cbp->cb_obp;
struct ccd_softc *cs = cbp->cb_sc;
int count;

#ifdef DEBUG
if (ccddebug & CCDB_FOLLOW)
printf("ccdiodone(%p)\n", cbp);
if (ccddebug & CCDB_IO) {
printf("ccdiodone: bp %p bcount %d resid %d\n",
bp, bp->b_bcount, bp->b_resid);
printf(" dev 0x%"PRIx64"(u%d), cbp %p bn %" PRId64 " addr %p"
" bcnt %d\n",
cbp->cb_buf.b_dev, cbp->cb_comp, cbp,
cbp->cb_buf.b_blkno, cbp->cb_buf.b_data,
cbp->cb_buf.b_bcount);
}
#endif

if (cbp->cb_buf.b_error != 0) {
bp->b_error = cbp->cb_buf.b_error;
printf("%s: error %d on component %d\n",
cs->sc_xname, bp->b_error, cbp->cb_comp);
}
count = cbp->cb_buf.b_bcount;
buf_destroy(&cbp->cb_buf);
CCD_PUTBUF(cbp);

/*
* If all done, "interrupt".
*/
mutex_enter(cs->sc_iolock);
bp->b_resid -= count;
if (bp->b_resid < 0)
panic("ccdiodone: count");
if (bp->b_resid == 0) {
/*
* Request is done for better or worse, wakeup the top half.
*/
if (bp->b_error != 0)
bp->b_resid = bp->b_bcount;
disk_unbusy(&cs->sc_dkdev, (bp->b_bcount - bp->b_resid),
(bp->b_flags & B_READ));
if (!disk_isbusy(&cs->sc_dkdev)) {
if (bufq_peek(cs->sc_bufq) != NULL) {
cv_broadcast(&cs->sc_push);
}
cv_broadcast(&cs->sc_stop);
}
mutex_exit(cs->sc_iolock);
biodone(bp);
} else
mutex_exit(cs->sc_iolock);
}

/* ARGSUSED */
static int
ccdread(dev_t dev, struct uio *uio, int flags)
{
int unit = ccdunit(dev);
struct ccd_softc *cs;

#ifdef DEBUG
if (ccddebug & CCDB_FOLLOW)
printf("ccdread(0x%"PRIx64", %p)\n", dev, uio);
#endif
if ((cs = ccdget(unit, 0)) == NULL)
return 0;

/* Unlocked advisory check, ccdstrategy check is synchronous. */
if ((cs->sc_flags & CCDF_INITED) == 0)
return (ENXIO);

return (physio(ccdstrategy, NULL, dev, B_READ, minphys, uio));
}

/* ARGSUSED */
static int
ccdwrite(dev_t dev, struct uio *uio, int flags)
{
int unit = ccdunit(dev);
struct ccd_softc *cs;

#ifdef DEBUG
if (ccddebug & CCDB_FOLLOW)
printf("ccdwrite(0x%"PRIx64", %p)\n", dev, uio);
#endif
if ((cs = ccdget(unit, 0)) == NULL)
return ENOENT;

/* Unlocked advisory check, ccdstrategy check is synchronous. */
if ((cs->sc_flags & CCDF_INITED) == 0)
return (ENXIO);

return (physio(ccdstrategy, NULL, dev, B_WRITE, minphys, uio));
}

int (*compat_ccd_ioctl_60)(dev_t, u_long, void *, int, struct lwp *,
int (*)(dev_t, u_long, void *, int, struct lwp *)) = (void *)enosys;

static int
ccdioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l)
{
int unit = ccdunit(dev);
int i, j, lookedup = 0, error = 0;
int part, pmask, make, hook;
struct ccd_softc *cs;
struct ccd_ioctl *ccio = (struct ccd_ioctl *)data;
kauth_cred_t uc;
char **cpp;
struct pathbuf *pb;
struct vnode **vpp;
#ifdef __HAVE_OLD_DISKLABEL
struct disklabel newlabel;
#endif

switch (cmd) {
case CCDIOCSET:
make = 1;
break;
default:
MODULE_HOOK_CALL(ccd_ioctl_60_hook,
(0, cmd, NULL, 0, NULL, NULL),
enosys(), hook);
if (hook == 0)
make = 1;
else
make = 0;
break;
}

if ((cs = ccdget(unit, make)) == NULL)
return ENOENT;
uc = kauth_cred_get();

MODULE_HOOK_CALL(ccd_ioctl_60_hook,
(dev, cmd, data, flag, l, ccdioctl),
enosys(), error);
if (error != ENOSYS)
return error;

/* Must be open for writes for these commands... */
switch (cmd) {
case CCDIOCSET:
case CCDIOCCLR:
case DIOCSDINFO:
case DIOCWDINFO:
case DIOCCACHESYNC:
case DIOCAWEDGE:
case DIOCDWEDGE:
case DIOCRMWEDGES:
case DIOCMWEDGES:
#ifdef __HAVE_OLD_DISKLABEL
case ODIOCSDINFO:
case ODIOCWDINFO:
#endif
case DIOCKLABEL:
case DIOCWLABEL:
if ((flag & FWRITE) == 0)
return (EBADF);
}

/* Must be initialized for these... */
switch (cmd) {
case CCDIOCCLR:
case DIOCGDINFO:
case DIOCGSTRATEGY:
case DIOCGCACHE:
case DIOCCACHESYNC:
case DIOCAWEDGE:
case DIOCDWEDGE:
case DIOCLWEDGES:
case DIOCMWEDGES:
case DIOCSDINFO:
case DIOCWDINFO:
case DIOCGPARTINFO:
case DIOCWLABEL:
case DIOCKLABEL:
case DIOCGDEFLABEL:
#ifdef __HAVE_OLD_DISKLABEL
case ODIOCGDINFO:
case ODIOCSDINFO:
case ODIOCWDINFO:
case ODIOCGDEFLABEL:
#endif
if ((cs->sc_flags & CCDF_INITED) == 0)
return ENXIO;
}

error = disk_ioctl(&cs->sc_dkdev, dev, cmd, data, flag, l);
if (error != EPASSTHROUGH)
return error;

switch (cmd) {
case DIOCGSTRATEGY:
{
struct disk_strategy *dks = (void *)data;

mutex_enter(cs->sc_iolock);
if (cs->sc_bufq != NULL)
strlcpy(dks->dks_name,
bufq_getstrategyname(cs->sc_bufq),
sizeof(dks->dks_name));
else
error = EINVAL;
mutex_exit(cs->sc_iolock);
dks->dks_paramlen = 0;
break;
}

case DIOCWDINFO:
case DIOCSDINFO:
#ifdef __HAVE_OLD_DISKLABEL
case ODIOCWDINFO:
case ODIOCSDINFO:
#endif
{
struct disklabel *lp;
#ifdef __HAVE_OLD_DISKLABEL
if (cmd == ODIOCSDINFO || cmd == ODIOCWDINFO) {
memset(&newlabel, 0, sizeof newlabel);
memcpy(&newlabel, data, sizeof (struct olddisklabel));
lp = &newlabel;
} else
#endif
lp = (struct disklabel *)data;

cs->sc_flags |= CCDF_LABELLING;

error = setdisklabel(cs->sc_dkdev.dk_label,
lp, 0, cs->sc_dkdev.dk_cpulabel);
if (error == 0) {
if (cmd == DIOCWDINFO
#ifdef __HAVE_OLD_DISKLABEL
|| cmd == ODIOCWDINFO
#endif
)
error = writedisklabel(CCDLABELDEV(dev),
ccdstrategy, cs->sc_dkdev.dk_label,
cs->sc_dkdev.dk_cpulabel);
}

cs->sc_flags &= ~CCDF_LABELLING;
break;
}

case DIOCKLABEL:
if (*(int *)data != 0)
cs->sc_flags |= CCDF_KLABEL;
else
cs->sc_flags &= ~CCDF_KLABEL;
break;

case DIOCWLABEL:
if (*(int *)data != 0)
cs->sc_flags |= CCDF_WLABEL;
else
cs->sc_flags &= ~CCDF_WLABEL;
break;

case DIOCGDEFLABEL:
ccdgetdefaultlabel(cs, (struct disklabel *)data);
break;

#ifdef __HAVE_OLD_DISKLABEL
case ODIOCGDEFLABEL:
ccdgetdefaultlabel(cs, &newlabel);
if (newlabel.d_npartitions > OLDMAXPARTITIONS)
return ENOTTY;
memcpy(data, &newlabel, sizeof (struct olddisklabel));
break;
#endif
default:
error = ENOTTY;
break;
}

if (error != ENOTTY)
return error;

mutex_enter(&cs->sc_dvlock);

error = 0;
switch (cmd) {
case CCDIOCSET:
if (cs->sc_flags & CCDF_INITED) {
error = EBUSY;
goto out;
}

/* Validate the flags. */
if ((ccio->ccio_flags & CCDF_USERMASK) != ccio->ccio_flags) {
error = EINVAL;
goto out;
}

if (ccio->ccio_ndisks > CCD_MAXNDISKS ||
ccio->ccio_ndisks == 0) {
error = EINVAL;
goto out;
}

/* Fill in some important bits. */
cs->sc_ileave = ccio->ccio_ileave;
cs->sc_nccdisks = ccio->ccio_ndisks;
cs->sc_flags = ccio->ccio_flags & CCDF_USERMASK;

/*
* Allocate space for and copy in the array of
* component pathnames and device numbers.
*/
cpp = kmem_alloc(ccio->ccio_ndisks * sizeof(*cpp), KM_SLEEP);
vpp = kmem_alloc(ccio->ccio_ndisks * sizeof(*vpp), KM_SLEEP);
error = copyin(ccio->ccio_disks, cpp,
ccio->ccio_ndisks * sizeof(*cpp));
if (error) {
kmem_free(vpp, ccio->ccio_ndisks * sizeof(*vpp));
kmem_free(cpp, ccio->ccio_ndisks * sizeof(*cpp));
goto out;
}

#ifdef DEBUG
if (ccddebug & CCDB_INIT)
for (i = 0; i < ccio->ccio_ndisks; ++i)
printf("ccdioctl: component %d: %p\n",
i, cpp[i]);
#endif

for (i = 0; i < ccio->ccio_ndisks; ++i) {
#ifdef DEBUG
if (ccddebug & CCDB_INIT)
printf("ccdioctl: lookedup = %d\n", lookedup);
#endif
error = pathbuf_copyin(cpp[i], &pb);
if (error == 0) {
error = vn_bdev_openpath(pb, &vpp[i], l);
pathbuf_destroy(pb);
}
if (error != 0) {
for (j = 0; j < lookedup; ++j)
(void)vn_close(vpp[j], FREAD|FWRITE,
uc);
kmem_free(vpp, ccio->ccio_ndisks *
sizeof(*vpp));
kmem_free(cpp, ccio->ccio_ndisks *
sizeof(*cpp));

/*
* No component data is allocated,
* nothing is to be freed.
*/
cs->sc_nccdisks = 0;
goto out;
}
++lookedup;
}

/* Attach the disk. */
disk_attach(&cs->sc_dkdev);
bufq_alloc(&cs->sc_bufq, "fcfs", 0);

/*
* Initialize the ccd. Fills in the softc for us.
*/
if ((error = ccdinit(cs, cpp, vpp, l)) != 0) {
for (j = 0; j < lookedup; ++j)
(void)vn_close(vpp[j], FREAD|FWRITE,
uc);
kmem_free(vpp, ccio->ccio_ndisks * sizeof(*vpp));
kmem_free(cpp, ccio->ccio_ndisks * sizeof(*cpp));
disk_detach(&cs->sc_dkdev);
mutex_exit(&cs->sc_dvlock);
bufq_free(cs->sc_bufq);
return error;
}

/* We can free the temporary variables now. */
kmem_free(vpp, ccio->ccio_ndisks * sizeof(*vpp));
kmem_free(cpp, ccio->ccio_ndisks * sizeof(*cpp));

/*
* The ccd has been successfully initialized, so
* we can place it into the array. Don't try to
* read the disklabel until the disk has been attached,
* because space for the disklabel is allocated
* in disk_attach();
*/
ccio->ccio_unit = unit;
ccio->ccio_size = cs->sc_size;

/* Try and read the disklabel. */
ccdgetdisklabel(dev);
disk_set_info(NULL, &cs->sc_dkdev, NULL);

/* discover wedges */
mutex_exit(&cs->sc_dvlock);
dkwedge_discover(&cs->sc_dkdev);
return 0;

case CCDIOCCLR:
/*
* Don't unconfigure if any other partitions are open
* or if both the character and block flavors of this
* partition are open.
*/
part = DISKPART(dev);
pmask = (1 << part);
if ((cs->sc_dkdev.dk_openmask & ~pmask) ||
((cs->sc_dkdev.dk_bopenmask & pmask) &&
(cs->sc_dkdev.dk_copenmask & pmask))) {
error = EBUSY;
goto out;
}

/* Delete all of our wedges. */
dkwedge_delall(&cs->sc_dkdev);

/* Stop new I/O, wait for in-flight I/O to complete. */
mutex_enter(cs->sc_iolock);
cs->sc_flags &= ~(CCDF_INITED|CCDF_VLABEL);
cs->sc_zap = true;
while (disk_isbusy(&cs->sc_dkdev) ||
bufq_peek(cs->sc_bufq) != NULL ||
cs->sc_thread != NULL) {
cv_broadcast(&cs->sc_push);
(void)cv_timedwait(&cs->sc_stop, cs->sc_iolock, hz);
}
mutex_exit(cs->sc_iolock);

/*
* Free ccd_softc information and clear entry.
*/

/* Close the components and free their pathnames. */
for (i = 0; i < cs->sc_nccdisks; ++i) {
/*
* XXX: this close could potentially fail and
* cause Bad Things. Maybe we need to force
* the close to happen?
*/
#ifdef DEBUG
if (ccddebug & CCDB_VNODE)
vprint("CCDIOCCLR: vnode info",
cs->sc_cinfo[i].ci_vp);
#endif
(void)vn_close(cs->sc_cinfo[i].ci_vp, FREAD|FWRITE,
uc);
kmem_free(cs->sc_cinfo[i].ci_path,
cs->sc_cinfo[i].ci_pathlen);
}

if (cs->sc_nccdisks != 0) {
/* Free interleave index. */
for (i = 0; cs->sc_itable[i].ii_ndisk; ++i) {
kmem_free(cs->sc_itable[i].ii_index,
cs->sc_itable[i].ii_indexsz);
}
/* Free component info and interleave table. */
kmem_free(cs->sc_cinfo, cs->sc_nccdisks *
sizeof(struct ccdcinfo));
kmem_free(cs->sc_itable, (cs->sc_nccdisks + 1) *
sizeof(struct ccdiinfo));
}

aprint_normal("%s: detached\n", cs->sc_xname);

/* Detach the disk. */
disk_detach(&cs->sc_dkdev);
bufq_free(cs->sc_bufq);

/* also releases sc_dvlock */
ccdput(cs);

/* Don't break, otherwise cs is read again. */
return 0;

case DIOCGCACHE:
{
int dkcache = 0;

/*
* We pass this call down to all components and report
* intersection of the flags returned by the components.
* If any errors out, we return error. CCD components
* can not change unless the device is unconfigured, so
* device feature flags will remain static. RCE/WCE can change
* of course, if set directly on underlying device.
*/
for (error = 0, i = 0; i < cs->sc_nccdisks; i++) {
error = VOP_IOCTL(cs->sc_cinfo[i].ci_vp, cmd, &j,
flag, uc);
if (error)
break;

if (i == 0)
dkcache = j;
else
dkcache = DKCACHE_COMBINE(dkcache, j);
}

*((int *)data) = dkcache;
break;
}

case DIOCCACHESYNC:
/*
* We pass this call down to all components and report
* the first error we encounter.
*/
for (error = 0, i = 0; i < cs->sc_nccdisks; i++) {
j = VOP_IOCTL(cs->sc_cinfo[i].ci_vp, cmd, data,
flag, uc);
if (j != 0 && error == 0)
error = j;
}
break;

default:
error = ENOTTY;
break;
}

out:
mutex_exit(&cs->sc_dvlock);
return (error);
}

static int
ccdsize(dev_t dev)
{
struct ccd_softc *cs;
struct disklabel *lp;
int part, unit, omask, size;

unit = ccdunit(dev);
if ((cs = ccdget(unit, 0)) == NULL)
return -1;

if ((cs->sc_flags & CCDF_INITED) == 0)
return (-1);

part = DISKPART(dev);
omask = cs->sc_dkdev.dk_openmask & (1 << part);
lp = cs->sc_dkdev.dk_label;

if (omask == 0 && ccdopen(dev, 0, S_IFBLK, curlwp))
return (-1);

if (lp->d_partitions[part].p_fstype != FS_SWAP)
size = -1;
else
size = lp->d_partitions[part].p_size *
(lp->d_secsize / DEV_BSIZE);

if (omask == 0 && ccdclose(dev, 0, S_IFBLK, curlwp))
return (-1);

return (size);
}

static void
ccdgetdefaultlabel(struct ccd_softc *cs, struct disklabel *lp)
{
struct ccdgeom *ccg = &cs->sc_geom;

memset(lp, 0, sizeof(*lp));

if (cs->sc_size > UINT32_MAX)
lp->d_secperunit = UINT32_MAX;
else
lp->d_secperunit = cs->sc_size;
lp->d_secsize = ccg->ccg_secsize;
lp->d_nsectors = ccg->ccg_nsectors;
lp->d_ntracks = ccg->ccg_ntracks;
lp->d_ncylinders = ccg->ccg_ncylinders;
lp->d_secpercyl = lp->d_ntracks * lp->d_nsectors;

strncpy(lp->d_typename, "ccd", sizeof(lp->d_typename));
lp->d_type = DKTYPE_CCD;
strncpy(lp->d_packname, "fictitious", sizeof(lp->d_packname));
lp->d_rpm = 3600;
lp->d_interleave = 1;
lp->d_flags = 0;

lp->d_partitions[RAW_PART].p_offset = 0;
lp->d_partitions[RAW_PART].p_size = lp->d_secperunit;
lp->d_partitions[RAW_PART].p_fstype = FS_UNUSED;
lp->d_npartitions = RAW_PART + 1;

lp->d_magic = DISKMAGIC;
lp->d_magic2 = DISKMAGIC;
lp->d_checksum = dkcksum(cs->sc_dkdev.dk_label);
}

/*
* Read the disklabel from the ccd. If one is not present, fake one
* up.
*/
static void
ccdgetdisklabel(dev_t dev)
{
int unit = ccdunit(dev);
struct ccd_softc *cs;
const char *errstring;
struct disklabel *lp;
struct cpu_disklabel *clp;

if ((cs = ccdget(unit, 0)) == NULL)
return;
lp = cs->sc_dkdev.dk_label;
clp = cs->sc_dkdev.dk_cpulabel;
KASSERT(mutex_owned(&cs->sc_dvlock));

memset(clp, 0, sizeof(*clp));

ccdgetdefaultlabel(cs, lp);

/*
* Call the generic disklabel extraction routine.
*/
cs->sc_flags |= CCDF_RLABEL;
if ((cs->sc_flags & CCDF_NOLABEL) != 0)
errstring = "CCDF_NOLABEL set; ignoring on-disk label";
else
errstring = readdisklabel(CCDLABELDEV(dev), ccdstrategy,
cs->sc_dkdev.dk_label, cs->sc_dkdev.dk_cpulabel);
if (errstring)
ccdmakedisklabel(cs);
else {
int i;
struct partition *pp;

/*
* Sanity check whether the found disklabel is valid.
*
* This is necessary since total size of ccd may vary
* when an interleave is changed even though exactly
* same components are used, and old disklabel may used
* if that is found.
*/
if (lp->d_secperunit < UINT32_MAX ?
lp->d_secperunit != cs->sc_size :
lp->d_secperunit > cs->sc_size)
printf("WARNING: %s: "
"total sector size in disklabel (%ju) != "
"the size of ccd (%ju)\n", cs->sc_xname,
(uintmax_t)lp->d_secperunit,
(uintmax_t)cs->sc_size);
for (i = 0; i < lp->d_npartitions; i++) {
pp = &lp->d_partitions[i];
if (pp->p_offset + pp->p_size > cs->sc_size)
printf("WARNING: %s: end of partition `%c' "
"exceeds the size of ccd (%ju)\n",
cs->sc_xname, 'a' + i, (uintmax_t)cs->sc_size);
}
}

#ifdef DEBUG
/* It's actually extremely common to have unlabeled ccds. */
if (ccddebug & CCDB_LABEL)
if (errstring != NULL)
printf("%s: %s\n", cs->sc_xname, errstring);
#endif

/* In-core label now valid. */
cs->sc_flags = (cs->sc_flags | CCDF_VLABEL) & ~CCDF_RLABEL;
}

/*
* Take care of things one might want to take care of in the event
* that a disklabel isn't present.
*/
static void
ccdmakedisklabel(struct ccd_softc *cs)
{
struct disklabel *lp = cs->sc_dkdev.dk_label;

/*
* For historical reasons, if there's no disklabel present
* the raw partition must be marked FS_BSDFFS.
*/
lp->d_partitions[RAW_PART].p_fstype = FS_BSDFFS;

strncpy(lp->d_packname, "default label", sizeof(lp->d_packname));

lp->d_checksum = dkcksum(lp);
}

#ifdef DEBUG
static void
printiinfo(struct ccdiinfo *ii)
{
int ix, i;

for (ix = 0; ii->ii_ndisk; ix++, ii++) {
printf(" itab[%d]: #dk %d sblk %" PRId64 " soff %" PRId64,
ix, ii->ii_ndisk, ii->ii_startblk, ii->ii_startoff);
for (i = 0; i < ii->ii_ndisk; i++)
printf(" %d", ii->ii_index[i]);
printf("\n");
}
}
#endif

MODULE(MODULE_CLASS_DRIVER, ccd, "dk_subr,bufq_fcfs");

static int
ccd_modcmd(modcmd_t cmd, void *arg)
{
int error = 0;
#ifdef _MODULE
int bmajor = -1, cmajor = -1;
#endif

switch (cmd) {
case MODULE_CMD_INIT:
#ifdef _MODULE
ccdattach(0);

error = devsw_attach("ccd", &ccd_bdevsw, &bmajor,
&ccd_cdevsw, &cmajor);
#endif
break;

case MODULE_CMD_FINI:
#ifdef _MODULE
mutex_enter(&ccd_lock);
if (!LIST_EMPTY(&ccds)) {
mutex_exit(&ccd_lock);
error = EBUSY;
} else {
mutex_exit(&ccd_lock);
devsw_detach(&ccd_bdevsw, &ccd_cdevsw);
ccddetach();
}
#endif
break;

case MODULE_CMD_STAT:
return ENOTTY;

default:
return ENOTTY;
}

return error;
}

static int
ccd_units_sysctl(SYSCTLFN_ARGS)
{
struct sysctlnode node;
struct ccd_softc *sc;
int error, i, nccd, *units;
size_t size;

nccd = 0;
mutex_enter(&ccd_lock);
LIST_FOREACH(sc, &ccds, sc_link)
nccd++;
mutex_exit(&ccd_lock);

if (nccd != 0) {
size = nccd * sizeof(*units);
units = kmem_zalloc(size, KM_SLEEP);
i = 0;
mutex_enter(&ccd_lock);
LIST_FOREACH(sc, &ccds, sc_link) {
if (i >= nccd)
break;
units[i] = sc->sc_unit;
}
mutex_exit(&ccd_lock);
} else {
units = NULL;
size = 0;
}

node = *rnode;
node.sysctl_data = units;
node.sysctl_size = size;

error = sysctl_lookup(SYSCTLFN_CALL(&node));
if (units)
kmem_free(units, size);
return error;
}

static int
ccd_info_sysctl(SYSCTLFN_ARGS)
{
struct sysctlnode node;
struct ccddiskinfo ccd;
struct ccd_softc *sc;
int unit, error;

if (newp == NULL || newlen != sizeof(int))
return EINVAL;

error = sysctl_copyin(l, newp, &unit, sizeof unit);
if (error)
return error;
newlen = 0;
ccd.ccd_ndisks = ~0;
mutex_enter(&ccd_lock);
LIST_FOREACH(sc, &ccds, sc_link) {
if (sc->sc_unit == unit) {
ccd.ccd_ileave = sc->sc_ileave;
ccd.ccd_size = sc->sc_size;
ccd.ccd_ndisks = sc->sc_nccdisks;
ccd.ccd_flags = sc->sc_flags;
break;
}
}
mutex_exit(&ccd_lock);

if (ccd.ccd_ndisks == ~0)
return ENOENT;

node = *rnode;
node.sysctl_data = &ccd;
node.sysctl_size = sizeof(ccd);

return sysctl_lookup(SYSCTLFN_CALL(&node));
}

static int
ccd_components_sysctl(SYSCTLFN_ARGS)
{
struct sysctlnode node;
int error, unit;
size_t size;
char *names, *p, *ep;
struct ccd_softc *sc;

if (newp == NULL || newlen != sizeof(int))
return EINVAL;

size = 0;
error = sysctl_copyin(l, newp, &unit, sizeof unit);
if (error)
return error;
newlen = 0;
mutex_enter(&ccd_lock);
LIST_FOREACH(sc, &ccds, sc_link)
if (sc->sc_unit == unit) {
for (size_t i = 0; i < sc->sc_nccdisks; i++)
size += strlen(sc->sc_cinfo[i].ci_path) + 1;
break;
}
mutex_exit(&ccd_lock);

if (size == 0)
return ENOENT;
names = kmem_zalloc(size, KM_SLEEP);
p = names;
ep = names + size;
mutex_enter(&ccd_lock);
LIST_FOREACH(sc, &ccds, sc_link)
if (sc->sc_unit == unit) {
for (size_t i = 0; i < sc->sc_nccdisks; i++) {
char *d = sc->sc_cinfo[i].ci_path;
while (p < ep && (*p++ = *d++) != '\0')
continue;
}
break;
}
mutex_exit(&ccd_lock);

node = *rnode;
node.sysctl_data = names;
node.sysctl_size = ep - names;

error = sysctl_lookup(SYSCTLFN_CALL(&node));
kmem_free(names, size);
return error;
}

SYSCTL_SETUP(sysctl_kern_ccd_setup, "sysctl kern.ccd subtree setup")
{
const struct sysctlnode *node = NULL;

sysctl_createv(clog, 0, NULL, &node,
CTLFLAG_PERMANENT,
CTLTYPE_NODE, "ccd",
SYSCTL_DESCR("ConCatenated Disk state"),
NULL, 0, NULL, 0,
CTL_KERN, CTL_CREATE, CTL_EOL);

if (node == NULL)
return;

sysctl_createv(clog, 0, &node, NULL,
CTLFLAG_PERMANENT | CTLFLAG_READONLY,
CTLTYPE_STRUCT, "units",
SYSCTL_DESCR("List of ccd unit numbers"),
ccd_units_sysctl, 0, NULL, 0,
CTL_CREATE, CTL_EOL);
sysctl_createv(clog, 0, &node, NULL,
CTLFLAG_PERMANENT | CTLFLAG_READWRITE,
CTLTYPE_STRUCT, "info",
SYSCTL_DESCR("Information about a CCD unit"),
ccd_info_sysctl, 0, NULL, 0,
CTL_CREATE, CTL_EOL);
sysctl_createv(clog, 0, &node, NULL,
CTLFLAG_PERMANENT | CTLFLAG_READWRITE,
CTLTYPE_STRUCT, "components",
SYSCTL_DESCR("Information about CCD components"),
ccd_components_sysctl, 0, NULL, 0,
CTL_CREATE, CTL_EOL);
}