* Copyright (c) 2002 The NetBSD Foundation, Inc.

/* $NetBSD: cgd.c,v 1.146 2022/04/02 09:53:20 riastradh Exp $ */

/*-
* Copyright (c) 2002 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
* by Roland C. Dowdeswell.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: cgd.c,v 1.146 2022/04/02 09:53:20 riastradh Exp $");

#include <sys/types.h>
#include <sys/param.h>
#include <sys/buf.h>
#include <sys/bufq.h>
#include <sys/conf.h>
#include <sys/cpu.h>
#include <sys/device.h>
#include <sys/disk.h>
#include <sys/disklabel.h>
#include <sys/errno.h>
#include <sys/fcntl.h>
#include <sys/ioctl.h>
#include <sys/kmem.h>
#include <sys/module.h>
#include <sys/namei.h> /* for pathbuf */
#include <sys/pool.h>
#include <sys/proc.h>
#include <sys/syslog.h>
#include <sys/systm.h>
#include <sys/vnode.h>
#include <sys/workqueue.h>

#include <dev/cgd_crypto.h>
#include <dev/cgdvar.h>
#include <dev/dkvar.h>

#include <miscfs/specfs/specdev.h> /* for v_rdev */

#include "ioconf.h"

struct selftest_params {
const char *alg;
int encblkno8;
int blocksize; /* number of bytes */
int secsize;
daddr_t blkno;
int keylen; /* number of bits */
int txtlen; /* number of bytes */
const uint8_t *key;
const uint8_t *ptxt;
const uint8_t *ctxt;
};

/* Entry Point Functions */

static dev_type_open(cgdopen);
static dev_type_close(cgdclose);
static dev_type_read(cgdread);
static dev_type_write(cgdwrite);
static dev_type_ioctl(cgdioctl);
static dev_type_strategy(cgdstrategy);
static dev_type_dump(cgddump);
static dev_type_size(cgdsize);

const struct bdevsw cgd_bdevsw = {
.d_open = cgdopen,
.d_close = cgdclose,
.d_strategy = cgdstrategy,
.d_ioctl = cgdioctl,
.d_dump = cgddump,
.d_psize = cgdsize,
.d_discard = nodiscard,
.d_flag = D_DISK | D_MPSAFE
};

const struct cdevsw cgd_cdevsw = {
.d_open = cgdopen,
.d_close = cgdclose,
.d_read = cgdread,
.d_write = cgdwrite,
.d_ioctl = cgdioctl,
.d_stop = nostop,
.d_tty = notty,
.d_poll = nopoll,
.d_mmap = nommap,
.d_kqfilter = nokqfilter,
.d_discard = nodiscard,
.d_flag = D_DISK | D_MPSAFE
};

/*
* Vector 5 from IEEE 1619/D16 truncated to 64 bytes, blkno 1.
*/
static const uint8_t selftest_aes_xts_256_ptxt[64] = {
0x27, 0xa7, 0x47, 0x9b, 0xef, 0xa1, 0xd4, 0x76,
0x48, 0x9f, 0x30, 0x8c, 0xd4, 0xcf, 0xa6, 0xe2,
0xa9, 0x6e, 0x4b, 0xbe, 0x32, 0x08, 0xff, 0x25,
0x28, 0x7d, 0xd3, 0x81, 0x96, 0x16, 0xe8, 0x9c,
0xc7, 0x8c, 0xf7, 0xf5, 0xe5, 0x43, 0x44, 0x5f,
0x83, 0x33, 0xd8, 0xfa, 0x7f, 0x56, 0x00, 0x00,
0x05, 0x27, 0x9f, 0xa5, 0xd8, 0xb5, 0xe4, 0xad,
0x40, 0xe7, 0x36, 0xdd, 0xb4, 0xd3, 0x54, 0x12,
};

static const uint8_t selftest_aes_xts_256_ctxt[512] = {
0x26, 0x4d, 0x3c, 0xa8, 0x51, 0x21, 0x94, 0xfe,
0xc3, 0x12, 0xc8, 0xc9, 0x89, 0x1f, 0x27, 0x9f,
0xef, 0xdd, 0x60, 0x8d, 0x0c, 0x02, 0x7b, 0x60,
0x48, 0x3a, 0x3f, 0xa8, 0x11, 0xd6, 0x5e, 0xe5,
0x9d, 0x52, 0xd9, 0xe4, 0x0e, 0xc5, 0x67, 0x2d,
0x81, 0x53, 0x2b, 0x38, 0xb6, 0xb0, 0x89, 0xce,
0x95, 0x1f, 0x0f, 0x9c, 0x35, 0x59, 0x0b, 0x8b,
0x97, 0x8d, 0x17, 0x52, 0x13, 0xf3, 0x29, 0xbb,
};

static const uint8_t selftest_aes_xts_256_key[33] = {
0x27, 0x18, 0x28, 0x18, 0x28, 0x45, 0x90, 0x45,
0x23, 0x53, 0x60, 0x28, 0x74, 0x71, 0x35, 0x26,
0x31, 0x41, 0x59, 0x26, 0x53, 0x58, 0x97, 0x93,
0x23, 0x84, 0x62, 0x64, 0x33, 0x83, 0x27, 0x95,
0
};

/*
* Vector 11 from IEEE 1619/D16 truncated to 64 bytes, blkno 0xffff.
*/
static const uint8_t selftest_aes_xts_512_ptxt[64] = {
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27,
0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f,
0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f,
};

static const uint8_t selftest_aes_xts_512_ctxt[64] = {
0x77, 0xa3, 0x12, 0x51, 0x61, 0x8a, 0x15, 0xe6,
0xb9, 0x2d, 0x1d, 0x66, 0xdf, 0xfe, 0x7b, 0x50,
0xb5, 0x0b, 0xad, 0x55, 0x23, 0x05, 0xba, 0x02,
0x17, 0xa6, 0x10, 0x68, 0x8e, 0xff, 0x7e, 0x11,
0xe1, 0xd0, 0x22, 0x54, 0x38, 0xe0, 0x93, 0x24,
0x2d, 0x6d, 0xb2, 0x74, 0xfd, 0xe8, 0x01, 0xd4,
0xca, 0xe0, 0x6f, 0x20, 0x92, 0xc7, 0x28, 0xb2,
0x47, 0x85, 0x59, 0xdf, 0x58, 0xe8, 0x37, 0xc2,
};

static const uint8_t selftest_aes_xts_512_key[65] = {
0x27, 0x18, 0x28, 0x18, 0x28, 0x45, 0x90, 0x45,
0x23, 0x53, 0x60, 0x28, 0x74, 0x71, 0x35, 0x26,
0x62, 0x49, 0x77, 0x57, 0x24, 0x70, 0x93, 0x69,
0x99, 0x59, 0x57, 0x49, 0x66, 0x96, 0x76, 0x27,
0x31, 0x41, 0x59, 0x26, 0x53, 0x58, 0x97, 0x93,
0x23, 0x84, 0x62, 0x64, 0x33, 0x83, 0x27, 0x95,
0x02, 0x88, 0x41, 0x97, 0x16, 0x93, 0x99, 0x37,
0x51, 0x05, 0x82, 0x09, 0x74, 0x94, 0x45, 0x92,
0
};

static const uint8_t selftest_aes_cbc_key[32] = {
0x27, 0x18, 0x28, 0x18, 0x28, 0x45, 0x90, 0x45,
0x23, 0x53, 0x60, 0x28, 0x74, 0x71, 0x35, 0x26,
0x62, 0x49, 0x77, 0x57, 0x24, 0x70, 0x93, 0x69,
0x99, 0x59, 0x57, 0x49, 0x66, 0x96, 0x76, 0x27,
};

static const uint8_t selftest_aes_cbc_128_ptxt[64] = {
0x27, 0xa7, 0x47, 0x9b, 0xef, 0xa1, 0xd4, 0x76,
0x48, 0x9f, 0x30, 0x8c, 0xd4, 0xcf, 0xa6, 0xe2,
0xa9, 0x6e, 0x4b, 0xbe, 0x32, 0x08, 0xff, 0x25,
0x28, 0x7d, 0xd3, 0x81, 0x96, 0x16, 0xe8, 0x9c,
0xc7, 0x8c, 0xf7, 0xf5, 0xe5, 0x43, 0x44, 0x5f,
0x83, 0x33, 0xd8, 0xfa, 0x7f, 0x56, 0x00, 0x00,
0x05, 0x27, 0x9f, 0xa5, 0xd8, 0xb5, 0xe4, 0xad,
0x40, 0xe7, 0x36, 0xdd, 0xb4, 0xd3, 0x54, 0x12,
};

static const uint8_t selftest_aes_cbc_128_ctxt[64] = { /* blkno=1 */
0x93, 0x94, 0x56, 0x36, 0x83, 0xbc, 0xff, 0xa4,
0xe0, 0x24, 0x34, 0x12, 0xbe, 0xfa, 0xb0, 0x7d,
0x88, 0x1e, 0xc5, 0x57, 0x55, 0x23, 0x05, 0x0c,
0x69, 0xa5, 0xc1, 0xda, 0x64, 0xee, 0x74, 0x10,
0xc2, 0xc5, 0xe6, 0x66, 0xd6, 0xa7, 0x49, 0x1c,
0x9d, 0x40, 0xb5, 0x0c, 0x9b, 0x6e, 0x1c, 0xe6,
0xb1, 0x7a, 0x1c, 0xe7, 0x5a, 0xfe, 0xf9, 0x2a,
0x78, 0xfa, 0xb7, 0x7b, 0x08, 0xdf, 0x8e, 0x51,
};

static const uint8_t selftest_aes_cbc_256_ptxt[64] = {
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27,
0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f,
0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f,
};

static const uint8_t selftest_aes_cbc_256_ctxt[64] = { /* blkno=0xffff */
0x6c, 0xa3, 0x15, 0x17, 0x51, 0x90, 0xe9, 0x69,
0x08, 0x36, 0x7b, 0xa6, 0xbb, 0xd1, 0x0b, 0x9e,
0xcd, 0x6b, 0x1e, 0xaf, 0xb6, 0x2e, 0x62, 0x7d,
0x8e, 0xde, 0xf0, 0xed, 0x0d, 0x44, 0xe7, 0x31,
0x26, 0xcf, 0xd5, 0x0b, 0x3e, 0x95, 0x59, 0x89,
0xdf, 0x5d, 0xd6, 0x9a, 0x00, 0x66, 0xcc, 0x7f,
0x45, 0xd3, 0x06, 0x58, 0xed, 0xef, 0x49, 0x47,
0x87, 0x89, 0x17, 0x7d, 0x08, 0x56, 0x50, 0xe1,
};

static const uint8_t selftest_3des_cbc_key[24] = {
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
};

static const uint8_t selftest_3des_cbc_ptxt[64] = {
0x27, 0xa7, 0x47, 0x9b, 0xef, 0xa1, 0xd4, 0x76,
0x48, 0x9f, 0x30, 0x8c, 0xd4, 0xcf, 0xa6, 0xe2,
0xa9, 0x6e, 0x4b, 0xbe, 0x32, 0x08, 0xff, 0x25,
0x28, 0x7d, 0xd3, 0x81, 0x96, 0x16, 0xe8, 0x9c,
0xc7, 0x8c, 0xf7, 0xf5, 0xe5, 0x43, 0x44, 0x5f,
0x83, 0x33, 0xd8, 0xfa, 0x7f, 0x56, 0x00, 0x00,
0x05, 0x27, 0x9f, 0xa5, 0xd8, 0xb5, 0xe4, 0xad,
0x40, 0xe7, 0x36, 0xdd, 0xb4, 0xd3, 0x54, 0x12,
};

static const uint8_t selftest_3des_cbc_ctxt[64] = {
0xa2, 0xfe, 0x81, 0xaa, 0x10, 0x6c, 0xea, 0xb9,
0x11, 0x58, 0x1f, 0x29, 0xb5, 0x86, 0x71, 0x56,
0xe9, 0x25, 0x1d, 0x07, 0xb1, 0x69, 0x59, 0x6c,
0x96, 0x80, 0xf7, 0x54, 0x38, 0xaa, 0xa7, 0xe4,
0xe8, 0x81, 0xf5, 0x00, 0xbb, 0x1c, 0x00, 0x3c,
0xba, 0x38, 0x45, 0x97, 0x4c, 0xcf, 0x84, 0x14,
0x46, 0x86, 0xd9, 0xf4, 0xc5, 0xe2, 0xf0, 0x54,
0xde, 0x41, 0xf6, 0xa1, 0xef, 0x1b, 0x0a, 0xea,
};

static const uint8_t selftest_bf_cbc_key[56] = {
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27,
0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f,
0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
};

static const uint8_t selftest_bf_cbc_ptxt[64] = {
0x27, 0xa7, 0x47, 0x9b, 0xef, 0xa1, 0xd4, 0x76,
0x48, 0x9f, 0x30, 0x8c, 0xd4, 0xcf, 0xa6, 0xe2,
0xa9, 0x6e, 0x4b, 0xbe, 0x32, 0x08, 0xff, 0x25,
0x28, 0x7d, 0xd3, 0x81, 0x96, 0x16, 0xe8, 0x9c,
0xc7, 0x8c, 0xf7, 0xf5, 0xe5, 0x43, 0x44, 0x5f,
0x83, 0x33, 0xd8, 0xfa, 0x7f, 0x56, 0x00, 0x00,
0x05, 0x27, 0x9f, 0xa5, 0xd8, 0xb5, 0xe4, 0xad,
0x40, 0xe7, 0x36, 0xdd, 0xb4, 0xd3, 0x54, 0x12,
};

static const uint8_t selftest_bf_cbc_ctxt[64] = {
0xec, 0xa2, 0xc0, 0x0e, 0xa9, 0x7f, 0x04, 0x1e,
0x2e, 0x4f, 0x64, 0x07, 0x67, 0x3e, 0xf4, 0x58,
0x61, 0x5f, 0xd3, 0x50, 0x5e, 0xd3, 0x4d, 0x34,
0xa0, 0x53, 0xbe, 0x47, 0x75, 0x69, 0x3b, 0x1f,
0x86, 0xf2, 0xae, 0x8b, 0xb7, 0x91, 0xda, 0xd4,
0x2b, 0xa5, 0x47, 0x9b, 0x7d, 0x13, 0x30, 0xdd,
0x7b, 0xad, 0x86, 0x57, 0x51, 0x11, 0x74, 0x42,
0xb8, 0xbf, 0x69, 0x17, 0x20, 0x0a, 0xf7, 0xda,
};

static const uint8_t selftest_aes_cbc_encblkno8_zero64[64];
static const uint8_t selftest_aes_cbc_encblkno8_ctxt[64] = {
0xa2, 0x06, 0x26, 0x26, 0xac, 0xdc, 0xe7, 0xcf,
0x47, 0x68, 0x24, 0x0e, 0xfa, 0x40, 0x44, 0x83,
0x07, 0xe1, 0xf4, 0x5d, 0x53, 0x47, 0xa0, 0xfe,
0xc0, 0x6e, 0x4e, 0xf8, 0x9d, 0x98, 0x63, 0xb8,
0x2c, 0x27, 0xfa, 0x3a, 0xd5, 0x40, 0xda, 0xdb,
0xe6, 0xc3, 0xe4, 0xfb, 0x85, 0x53, 0xfb, 0x78,
0x5d, 0xbd, 0x8f, 0x4c, 0x1a, 0x04, 0x9c, 0x88,
0x85, 0xec, 0x3c, 0x56, 0x46, 0x1a, 0x6e, 0xf5,
};

const struct selftest_params selftests[] = {
{
.alg = "aes-xts",
.blocksize = 16,
.secsize = 512,
.blkno = 1,
.keylen = 256,
.txtlen = sizeof(selftest_aes_xts_256_ptxt),
.key = selftest_aes_xts_256_key,
.ptxt = selftest_aes_xts_256_ptxt,
.ctxt = selftest_aes_xts_256_ctxt
},
{
.alg = "aes-xts",
.blocksize = 16,
.secsize = 512,
.blkno = 0xffff,
.keylen = 512,
.txtlen = sizeof(selftest_aes_xts_512_ptxt),
.key = selftest_aes_xts_512_key,
.ptxt = selftest_aes_xts_512_ptxt,
.ctxt = selftest_aes_xts_512_ctxt
},
{
.alg = "aes-cbc",
.blocksize = 16,
.secsize = 512,
.blkno = 1,
.keylen = 128,
.txtlen = sizeof(selftest_aes_cbc_128_ptxt),
.key = selftest_aes_cbc_key,
.ptxt = selftest_aes_cbc_128_ptxt,
.ctxt = selftest_aes_cbc_128_ctxt,
},
{
.alg = "aes-cbc",
.blocksize = 16,
.secsize = 512,
.blkno = 0xffff,
.keylen = 256,
.txtlen = sizeof(selftest_aes_cbc_256_ptxt),
.key = selftest_aes_cbc_key,
.ptxt = selftest_aes_cbc_256_ptxt,
.ctxt = selftest_aes_cbc_256_ctxt,
},
{
.alg = "3des-cbc",
.blocksize = 8,
.secsize = 512,
.blkno = 1,
.keylen = 192, /* 168 + 3*8 parity bits */
.txtlen = sizeof(selftest_3des_cbc_ptxt),
.key = selftest_3des_cbc_key,
.ptxt = selftest_3des_cbc_ptxt,
.ctxt = selftest_3des_cbc_ctxt,
},
{
.alg = "blowfish-cbc",
.blocksize = 8,
.secsize = 512,
.blkno = 1,
.keylen = 448,
.txtlen = sizeof(selftest_bf_cbc_ptxt),
.key = selftest_bf_cbc_key,
.ptxt = selftest_bf_cbc_ptxt,
.ctxt = selftest_bf_cbc_ctxt,
},
{
.alg = "aes-cbc",
.encblkno8 = 1,
.blocksize = 16,
.secsize = 512,
.blkno = 0,
.keylen = 128,
.txtlen = sizeof(selftest_aes_cbc_encblkno8_zero64),
.key = selftest_aes_cbc_encblkno8_zero64,
.ptxt = selftest_aes_cbc_encblkno8_zero64,
.ctxt = selftest_aes_cbc_encblkno8_ctxt,
},
};

static int cgd_match(device_t, cfdata_t, void *);
static void cgd_attach(device_t, device_t, void *);
static int cgd_detach(device_t, int);
static struct cgd_softc *cgd_spawn(int);
static struct cgd_worker *cgd_create_one_worker(void);
static void cgd_destroy_one_worker(struct cgd_worker *);
static struct cgd_worker *cgd_create_worker(void);
static void cgd_destroy_worker(struct cgd_worker *);
static int cgd_destroy(device_t);

/* Internal Functions */

static int cgd_diskstart(device_t, struct buf *);
static void cgd_diskstart2(struct cgd_softc *, struct cgd_xfer *);
static void cgdiodone(struct buf *);
static void cgd_iodone2(struct cgd_softc *, struct cgd_xfer *);
static void cgd_enqueue(struct cgd_softc *, struct cgd_xfer *);
static void cgd_process(struct work *, void *);
static int cgd_dumpblocks(device_t, void *, daddr_t, int);

static int cgd_ioctl_set(struct cgd_softc *, void *, struct lwp *);
static int cgd_ioctl_clr(struct cgd_softc *, struct lwp *);
static int cgd_ioctl_get(dev_t, void *, struct lwp *);
static int cgdinit(struct cgd_softc *, const char *, struct vnode *,
struct lwp *);
static void cgd_cipher(struct cgd_softc *, void *, const void *,
size_t, daddr_t, size_t, int);

static void cgd_selftest(void);

static const struct dkdriver cgddkdriver = {
.d_minphys = minphys,
.d_open = cgdopen,
.d_close = cgdclose,
.d_strategy = cgdstrategy,
.d_iosize = NULL,
.d_diskstart = cgd_diskstart,
.d_dumpblocks = cgd_dumpblocks,
.d_lastclose = NULL
};

CFATTACH_DECL3_NEW(cgd, sizeof(struct cgd_softc),
cgd_match, cgd_attach, cgd_detach, NULL, NULL, NULL, DVF_DETACH_SHUTDOWN);

/* DIAGNOSTIC and DEBUG definitions */

#if defined(CGDDEBUG) && !defined(DEBUG)
#define DEBUG
#endif

#ifdef DEBUG
int cgddebug = 0;

#define CGDB_FOLLOW 0x1
#define CGDB_IO 0x2
#define CGDB_CRYPTO 0x4

#define IFDEBUG(x,y) if (cgddebug & (x)) y
#define DPRINTF(x,y) IFDEBUG(x, printf y)
#define DPRINTF_FOLLOW(y) DPRINTF(CGDB_FOLLOW, y)

static void hexprint(const char *, void *, int);

#else
#define IFDEBUG(x,y)
#define DPRINTF(x,y)
#define DPRINTF_FOLLOW(y)
#endif

/* Global variables */

static kmutex_t cgd_spawning_mtx;
static kcondvar_t cgd_spawning_cv;
static bool cgd_spawning;
static struct cgd_worker *cgd_worker;
static u_int cgd_refcnt; /* number of users of cgd_worker */

/* Utility Functions */

#define CGDUNIT(x) DISKUNIT(x)

/* The code */

static int
cgd_lock(bool intr)
{
int error = 0;

mutex_enter(&cgd_spawning_mtx);
while (cgd_spawning) {
if (intr)
error = cv_wait_sig(&cgd_spawning_cv, &cgd_spawning_mtx);
else
cv_wait(&cgd_spawning_cv, &cgd_spawning_mtx);
}
if (error == 0)
cgd_spawning = true;
mutex_exit(&cgd_spawning_mtx);
return error;
}

static void
cgd_unlock(void)
{
mutex_enter(&cgd_spawning_mtx);
cgd_spawning = false;
cv_broadcast(&cgd_spawning_cv);
mutex_exit(&cgd_spawning_mtx);
}

static struct cgd_softc *
getcgd_softc(dev_t dev)
{
return device_lookup_private(&cgd_cd, CGDUNIT(dev));
}

static int
cgd_match(device_t self, cfdata_t cfdata, void *aux)
{

return 1;
}

static void
cgd_attach(device_t parent, device_t self, void *aux)
{
struct cgd_softc *sc = device_private(self);

mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_BIO);
cv_init(&sc->sc_cv, "cgdcv");
dk_init(&sc->sc_dksc, self, DKTYPE_CGD);
disk_init(&sc->sc_dksc.sc_dkdev, sc->sc_dksc.sc_xname, &cgddkdriver);

if (!pmf_device_register(self, NULL, NULL))
aprint_error_dev(self,
"unable to register power management hooks\n");
}

static int
cgd_detach(device_t self, int flags)
{
int ret;
struct cgd_softc *sc = device_private(self);
struct dk_softc *dksc = &sc->sc_dksc;

if (DK_BUSY(dksc, 0))
return EBUSY;

if (DK_ATTACHED(dksc) &&
(ret = cgd_ioctl_clr(sc, curlwp)) != 0)
return ret;

disk_destroy(&dksc->sc_dkdev);
cv_destroy(&sc->sc_cv);
mutex_destroy(&sc->sc_lock);

return 0;
}

void
cgdattach(int num)
{
#ifndef _MODULE
int error;

mutex_init(&cgd_spawning_mtx, MUTEX_DEFAULT, IPL_NONE);
cv_init(&cgd_spawning_cv, "cgspwn");

error = config_cfattach_attach(cgd_cd.cd_name, &cgd_ca);
if (error != 0)
aprint_error("%s: unable to register cfattach\n",
cgd_cd.cd_name);
#endif

cgd_selftest();
}

static struct cgd_softc *
cgd_spawn(int unit)
{
cfdata_t cf;
struct cgd_worker *cw;
struct cgd_softc *sc;

cf = kmem_alloc(sizeof(*cf), KM_SLEEP);
cf->cf_name = cgd_cd.cd_name;
cf->cf_atname = cgd_cd.cd_name;
cf->cf_unit = unit;
cf->cf_fstate = FSTATE_STAR;

cw = cgd_create_one_worker();
if (cw == NULL) {
kmem_free(cf, sizeof(*cf));
return NULL;
}

sc = device_private(config_attach_pseudo(cf));
if (sc == NULL) {
cgd_destroy_one_worker(cw);
return NULL;
}

sc->sc_worker = cw;

return sc;
}

static int
cgd_destroy(device_t dev)
{
struct cgd_softc *sc = device_private(dev);
struct cgd_worker *cw = sc->sc_worker;
cfdata_t cf;
int error;

cf = device_cfdata(dev);
error = config_detach(dev, DETACH_QUIET);
if (error)
return error;

cgd_destroy_one_worker(cw);

kmem_free(cf, sizeof(*cf));
return 0;
}

static void
cgd_busy(struct cgd_softc *sc)
{

mutex_enter(&sc->sc_lock);
while (sc->sc_busy)
cv_wait(&sc->sc_cv, &sc->sc_lock);
sc->sc_busy = true;
mutex_exit(&sc->sc_lock);
}

static void
cgd_unbusy(struct cgd_softc *sc)
{

mutex_enter(&sc->sc_lock);
sc->sc_busy = false;
cv_broadcast(&sc->sc_cv);
mutex_exit(&sc->sc_lock);
}

static struct cgd_worker *
cgd_create_one_worker(void)
{
KASSERT(cgd_spawning);

if (cgd_refcnt++ == 0) {
KASSERT(cgd_worker == NULL);
cgd_worker = cgd_create_worker();
}

KASSERT(cgd_worker != NULL);
return cgd_worker;
}

static void
cgd_destroy_one_worker(struct cgd_worker *cw)
{
KASSERT(cgd_spawning);
KASSERT(cw == cgd_worker);

if (--cgd_refcnt == 0) {
cgd_destroy_worker(cgd_worker);
cgd_worker = NULL;
}
}

static struct cgd_worker *
cgd_create_worker(void)
{
struct cgd_worker *cw;
struct workqueue *wq;
struct pool *cp;
int error;

cw = kmem_alloc(sizeof(struct cgd_worker), KM_SLEEP);
cp = kmem_alloc(sizeof(struct pool), KM_SLEEP);

error = workqueue_create(&wq, "cgd", cgd_process, NULL,
PRI_BIO, IPL_BIO, WQ_FPU|WQ_MPSAFE|WQ_PERCPU);
if (error) {
kmem_free(cp, sizeof(struct pool));
kmem_free(cw, sizeof(struct cgd_worker));
return NULL;
}

cw->cw_cpool = cp;
cw->cw_wq = wq;
pool_init(cw->cw_cpool, sizeof(struct cgd_xfer), 0,
0, 0, "cgdcpl", NULL, IPL_BIO);
mutex_init(&cw->cw_lock, MUTEX_DEFAULT, IPL_BIO);

return cw;
}

static void
cgd_destroy_worker(struct cgd_worker *cw)
{

/*
* Wait for all worker threads to complete before destroying
* the rest of the cgd_worker.
*/
if (cw->cw_wq)
workqueue_destroy(cw->cw_wq);

mutex_destroy(&cw->cw_lock);

if (cw->cw_cpool) {
pool_destroy(cw->cw_cpool);
kmem_free(cw->cw_cpool, sizeof(struct pool));
}

kmem_free(cw, sizeof(struct cgd_worker));
}

static int
cgdopen(dev_t dev, int flags, int fmt, struct lwp *l)
{
struct cgd_softc *sc;
int error;

DPRINTF_FOLLOW(("cgdopen(0x%"PRIx64", %d)\n", dev, flags));

error = cgd_lock(true);
if (error)
return error;
sc = getcgd_softc(dev);
if (sc == NULL)
sc = cgd_spawn(CGDUNIT(dev));
cgd_unlock();
if (sc == NULL)
return ENXIO;

return dk_open(&sc->sc_dksc, dev, flags, fmt, l);
}

static int
cgdclose(dev_t dev, int flags, int fmt, struct lwp *l)
{
struct cgd_softc *sc;
struct dk_softc *dksc;
int error;

DPRINTF_FOLLOW(("cgdclose(0x%"PRIx64", %d)\n", dev, flags));

error = cgd_lock(false);
if (error)
return error;
sc = getcgd_softc(dev);
if (sc == NULL) {
error = ENXIO;
goto done;
}

dksc = &sc->sc_dksc;
if ((error = dk_close(dksc, dev, flags, fmt, l)) != 0)
goto done;

if (!DK_ATTACHED(dksc)) {
if ((error = cgd_destroy(sc->sc_dksc.sc_dev)) != 0) {
device_printf(dksc->sc_dev,
"unable to detach instance\n");
goto done;
}
}

done:
cgd_unlock();

return error;
}

static void
cgdstrategy(struct buf *bp)
{
struct cgd_softc *sc = getcgd_softc(bp->b_dev);

DPRINTF_FOLLOW(("cgdstrategy(%p): b_bcount = %ld\n", bp,
(long)bp->b_bcount));

/*
* Reject unaligned writes.
*/
if (((uintptr_t)bp->b_data & 3) != 0) {
bp->b_error = EINVAL;
goto bail;
}

dk_strategy(&sc->sc_dksc, bp);
return;

bail:
bp->b_resid = bp->b_bcount;
biodone(bp);
return;
}

static int
cgdsize(dev_t dev)
{
struct cgd_softc *sc = getcgd_softc(dev);

DPRINTF_FOLLOW(("cgdsize(0x%"PRIx64")\n", dev));
if (!sc)
return -1;
return dk_size(&sc->sc_dksc, dev);
}

/*
* cgd_{get,put}data are functions that deal with getting a buffer
* for the new encrypted data.
* We can no longer have a buffer per device, we need a buffer per
* work queue...
*/

static void *
cgd_getdata(struct cgd_softc *sc, unsigned long size)
{
void *data = NULL;

mutex_enter(&sc->sc_lock);
if (!sc->sc_data_used) {
sc->sc_data_used = true;
data = sc->sc_data;
}
mutex_exit(&sc->sc_lock);

if (data)
return data;

return kmem_intr_alloc(size, KM_NOSLEEP);
}

static void
cgd_putdata(struct cgd_softc *sc, void *data, unsigned long size)
{

if (data == sc->sc_data) {
mutex_enter(&sc->sc_lock);
sc->sc_data_used = false;
mutex_exit(&sc->sc_lock);
} else
kmem_intr_free(data, size);
}

static int
cgd_diskstart(device_t dev, struct buf *bp)
{
struct cgd_softc *sc = device_private(dev);
struct cgd_worker *cw = sc->sc_worker;
struct dk_softc *dksc = &sc->sc_dksc;
struct disk_geom *dg = &dksc->sc_dkdev.dk_geom;
struct cgd_xfer *cx;
struct buf *nbp;
void * newaddr;
daddr_t bn;

DPRINTF_FOLLOW(("cgd_diskstart(%p, %p)\n", dksc, bp));

bn = bp->b_rawblkno;

/*
* We attempt to allocate all of our resources up front, so that
* we can fail quickly if they are unavailable.
*/
nbp = getiobuf(sc->sc_tvn, false);
if (nbp == NULL)
return EAGAIN;

cx = pool_get(cw->cw_cpool, PR_NOWAIT);
if (cx == NULL) {
putiobuf(nbp);
return EAGAIN;
}

cx->cx_sc = sc;
cx->cx_obp = bp;
cx->cx_nbp = nbp;
cx->cx_srcv = cx->cx_dstv = bp->b_data;
cx->cx_blkno = bn;
cx->cx_secsize = dg->dg_secsize;

/*
* If we are writing, then we need to encrypt the outgoing
* block into a new block of memory.
*/
if ((bp->b_flags & B_READ) == 0) {
newaddr = cgd_getdata(sc, bp->b_bcount);
if (!newaddr) {
pool_put(cw->cw_cpool, cx);
putiobuf(nbp);
return EAGAIN;
}

cx->cx_dstv = newaddr;
cx->cx_len = bp->b_bcount;
cx->cx_dir = CGD_CIPHER_ENCRYPT;

cgd_enqueue(sc, cx);
return 0;
}

cgd_diskstart2(sc, cx);
return 0;
}

static void
cgd_diskstart2(struct cgd_softc *sc, struct cgd_xfer *cx)
{
struct vnode *vp;
struct buf *bp;
struct buf *nbp;

bp = cx->cx_obp;
nbp = cx->cx_nbp;

nbp->b_data = cx->cx_dstv;
nbp->b_flags = bp->b_flags;
nbp->b_oflags = bp->b_oflags;
nbp->b_cflags = bp->b_cflags;
nbp->b_iodone = cgdiodone;
nbp->b_proc = bp->b_proc;
nbp->b_blkno = btodb(cx->cx_blkno * cx->cx_secsize);
nbp->b_bcount = bp->b_bcount;
nbp->b_private = cx;

BIO_COPYPRIO(nbp, bp);

if ((nbp->b_flags & B_READ) == 0) {
vp = nbp->b_vp;
mutex_enter(vp->v_interlock);
vp->v_numoutput++;
mutex_exit(vp->v_interlock);
}
VOP_STRATEGY(sc->sc_tvn, nbp);
}

static void
cgdiodone(struct buf *nbp)
{
struct cgd_xfer *cx = nbp->b_private;
struct buf *obp = cx->cx_obp;
struct cgd_softc *sc = getcgd_softc(obp->b_dev);
struct dk_softc *dksc = &sc->sc_dksc;
struct disk_geom *dg = &dksc->sc_dkdev.dk_geom;
daddr_t bn;

KDASSERT(sc);

DPRINTF_FOLLOW(("cgdiodone(%p)\n", nbp));
DPRINTF(CGDB_IO, ("cgdiodone: bp %p bcount %d resid %d\n",
obp, obp->b_bcount, obp->b_resid));
DPRINTF(CGDB_IO, (" dev 0x%"PRIx64", nbp %p bn %" PRId64
" addr %p bcnt %d\n", nbp->b_dev, nbp, nbp->b_blkno, nbp->b_data,
nbp->b_bcount));
if (nbp->b_error != 0) {
obp->b_error = nbp->b_error;
DPRINTF(CGDB_IO, ("%s: error %d\n", dksc->sc_xname,
obp->b_error));
}

/* Perform the decryption if we are reading.
*
* Note: use the blocknumber from nbp, since it is what
* we used to encrypt the blocks.
*/

if (nbp->b_flags & B_READ) {
bn = dbtob(nbp->b_blkno) / dg->dg_secsize;

cx->cx_obp = obp;
cx->cx_nbp = nbp;
cx->cx_dstv = obp->b_data;
cx->cx_srcv = obp->b_data;
cx->cx_len = obp->b_bcount;
cx->cx_blkno = bn;
cx->cx_secsize = dg->dg_secsize;
cx->cx_dir = CGD_CIPHER_DECRYPT;

cgd_enqueue(sc, cx);
return;
}

cgd_iodone2(sc, cx);
}

static void
cgd_iodone2(struct cgd_softc *sc, struct cgd_xfer *cx)
{
struct cgd_worker *cw = sc->sc_worker;
struct buf *obp = cx->cx_obp;
struct buf *nbp = cx->cx_nbp;
struct dk_softc *dksc = &sc->sc_dksc;

pool_put(cw->cw_cpool, cx);

/* If we allocated memory, free it now... */
if (nbp->b_data != obp->b_data)
cgd_putdata(sc, nbp->b_data, nbp->b_bcount);

putiobuf(nbp);

/* Request is complete for whatever reason */
obp->b_resid = 0;
if (obp->b_error != 0)
obp->b_resid = obp->b_bcount;

dk_done(dksc, obp);
dk_start(dksc, NULL);
}

static int
cgd_dumpblocks(device_t dev, void *va, daddr_t blkno, int nblk)
{
struct cgd_softc *sc = device_private(dev);
struct dk_softc *dksc = &sc->sc_dksc;
struct disk_geom *dg = &dksc->sc_dkdev.dk_geom;
size_t nbytes, blksize;
void *buf;
int error;

/*
* dk_dump gives us units of disklabel sectors. Everything
* else in cgd uses units of diskgeom sectors. These had
* better agree; otherwise we need to figure out how to convert
* between them.
*/
KASSERTMSG((dg->dg_secsize == dksc->sc_dkdev.dk_label->d_secsize),
"diskgeom secsize %"PRIu32" != disklabel secsize %"PRIu32,
dg->dg_secsize, dksc->sc_dkdev.dk_label->d_secsize);
blksize = dg->dg_secsize;

/*
* Compute the number of bytes in this request, which dk_dump
* has `helpfully' converted to a number of blocks for us.
*/
nbytes = nblk*blksize;

/* Try to acquire a buffer to store the ciphertext. */
buf = cgd_getdata(sc, nbytes);
if (buf == NULL)
/* Out of memory: give up. */
return ENOMEM;

/* Encrypt the caller's data into the temporary buffer. */
cgd_cipher(sc, buf, va, nbytes, blkno, blksize, CGD_CIPHER_ENCRYPT);

/* Pass it on to the underlying disk device. */
error = bdev_dump(sc->sc_tdev, blkno, buf, nbytes);

/* Release the buffer. */
cgd_putdata(sc, buf, nbytes);

/* Return any error from the underlying disk device. */
return error;
}

/* XXX: we should probably put these into dksubr.c, mostly */
static int
cgdread(dev_t dev, struct uio *uio, int flags)
{
struct cgd_softc *sc;
struct dk_softc *dksc;

DPRINTF_FOLLOW(("cgdread(0x%llx, %p, %d)\n",
(unsigned long long)dev, uio, flags));
sc = getcgd_softc(dev);
if (sc == NULL)
return ENXIO;
dksc = &sc->sc_dksc;
if (!DK_ATTACHED(dksc))
return ENXIO;
return physio(cgdstrategy, NULL, dev, B_READ, minphys, uio);
}

/* XXX: we should probably put these into dksubr.c, mostly */
static int
cgdwrite(dev_t dev, struct uio *uio, int flags)
{
struct cgd_softc *sc;
struct dk_softc *dksc;

DPRINTF_FOLLOW(("cgdwrite(0x%"PRIx64", %p, %d)\n", dev, uio, flags));
sc = getcgd_softc(dev);
if (sc == NULL)
return ENXIO;
dksc = &sc->sc_dksc;
if (!DK_ATTACHED(dksc))
return ENXIO;
return physio(cgdstrategy, NULL, dev, B_WRITE, minphys, uio);
}

static int
cgdioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l)
{
struct cgd_softc *sc;
struct dk_softc *dksc;
int part = DISKPART(dev);
int pmask = 1 << part;
int error;

DPRINTF_FOLLOW(("cgdioctl(0x%"PRIx64", %ld, %p, %d, %p)\n",
dev, cmd, data, flag, l));

switch (cmd) {
case CGDIOCGET:
return cgd_ioctl_get(dev, data, l);
case CGDIOCSET:
case CGDIOCCLR:
if ((flag & FWRITE) == 0)
return EBADF;
/* FALLTHROUGH */
default:
sc = getcgd_softc(dev);
if (sc == NULL)
return ENXIO;
dksc = &sc->sc_dksc;
break;
}

switch (cmd) {
case CGDIOCSET:
cgd_busy(sc);
if (DK_ATTACHED(dksc))
error = EBUSY;
else
error = cgd_ioctl_set(sc, data, l);
cgd_unbusy(sc);
break;
case CGDIOCCLR:
cgd_busy(sc);
if (DK_BUSY(&sc->sc_dksc, pmask))
error = EBUSY;
else
error = cgd_ioctl_clr(sc, l);
cgd_unbusy(sc);
break;
case DIOCGCACHE:
case DIOCCACHESYNC:
cgd_busy(sc);
if (!DK_ATTACHED(dksc)) {
cgd_unbusy(sc);
error = ENOENT;
break;
}
/*
* We pass this call down to the underlying disk.
*/
error = VOP_IOCTL(sc->sc_tvn, cmd, data, flag, l->l_cred);
cgd_unbusy(sc);
break;
case DIOCGSECTORALIGN: {
struct disk_sectoralign *dsa = data;

cgd_busy(sc);
if (!DK_ATTACHED(dksc)) {
cgd_unbusy(sc);
error = ENOENT;
break;
}

/* Get the underlying disk's sector alignment. */
error = VOP_IOCTL(sc->sc_tvn, cmd, data, flag, l->l_cred);
if (error) {
cgd_unbusy(sc);
break;
}

/* Adjust for the disklabel partition if necessary. */
if (part != RAW_PART) {
struct disklabel *lp = dksc->sc_dkdev.dk_label;
daddr_t offset = lp->d_partitions[part].p_offset;
uint32_t r = offset % dsa->dsa_alignment;

if (r < dsa->dsa_firstaligned)
dsa->dsa_firstaligned = dsa->dsa_firstaligned
- r;
else
dsa->dsa_firstaligned = (dsa->dsa_firstaligned
+ dsa->dsa_alignment) - r;
}
cgd_unbusy(sc);
break;
}
case DIOCGSTRATEGY:
case DIOCSSTRATEGY:
if (!DK_ATTACHED(dksc)) {
error = ENOENT;
break;
}
/*FALLTHROUGH*/
default:
error = dk_ioctl(dksc, dev, cmd, data, flag, l);
break;
case CGDIOCGET:
KASSERT(0);
error = EINVAL;
}

return error;
}

static int
cgddump(dev_t dev, daddr_t blkno, void *va, size_t size)
{
struct cgd_softc *sc;

DPRINTF_FOLLOW(("cgddump(0x%"PRIx64", %" PRId64 ", %p, %lu)\n",
dev, blkno, va, (unsigned long)size));
sc = getcgd_softc(dev);
if (sc == NULL)
return ENXIO;
return dk_dump(&sc->sc_dksc, dev, blkno, va, size, DK_DUMP_RECURSIVE);
}

/*
* XXXrcd:
* for now we hardcode the maximum key length.
*/
#define MAX_KEYSIZE 1024

static const struct {
const char *n;
int v;
int d;
} encblkno[] = {
{ "encblkno", CGD_CIPHER_CBC_ENCBLKNO8, 1 },
{ "encblkno8", CGD_CIPHER_CBC_ENCBLKNO8, 1 },
{ "encblkno1", CGD_CIPHER_CBC_ENCBLKNO1, 8 },
};

/* ARGSUSED */
static int
cgd_ioctl_set(struct cgd_softc *sc, void *data, struct lwp *l)
{
struct cgd_ioctl *ci = data;
struct vnode *vp;
int ret;
size_t i;
size_t keybytes; /* key length in bytes */
const char *cp;
struct pathbuf *pb;
char *inbuf;
struct dk_softc *dksc = &sc->sc_dksc;

cp = ci->ci_disk;

ret = pathbuf_copyin(ci->ci_disk, &pb);
if (ret != 0) {
return ret;
}
ret = vn_bdev_openpath(pb, &vp, l);
pathbuf_destroy(pb);
if (ret != 0) {
return ret;
}

inbuf = kmem_alloc(MAX_KEYSIZE, KM_SLEEP);

if ((ret = cgdinit(sc, cp, vp, l)) != 0)
goto bail;

(void)memset(inbuf, 0, MAX_KEYSIZE);
ret = copyinstr(ci->ci_alg, inbuf, 256, NULL);
if (ret)
goto bail;
sc->sc_cfuncs = cryptfuncs_find(inbuf);
if (!sc->sc_cfuncs) {
ret = EINVAL;
goto bail;
}

(void)memset(inbuf, 0, MAX_KEYSIZE);
ret = copyinstr(ci->ci_ivmethod, inbuf, MAX_KEYSIZE, NULL);
if (ret)
goto bail;

for (i = 0; i < __arraycount(encblkno); i++)
if (strcmp(encblkno[i].n, inbuf) == 0)
break;

if (i == __arraycount(encblkno)) {
ret = EINVAL;
goto bail;
}

keybytes = ci->ci_keylen / 8 + 1;
if (keybytes > MAX_KEYSIZE) {
ret = EINVAL;
goto bail;
}

(void)memset(inbuf, 0, MAX_KEYSIZE);
ret = copyin(ci->ci_key, inbuf, keybytes);
if (ret)
goto bail;

sc->sc_cdata.cf_blocksize = ci->ci_blocksize;
sc->sc_cdata.cf_mode = encblkno[i].v;

/*
* Print a warning if the user selected the legacy encblkno8
* mistake, and reject it altogether for ciphers that it
* doesn't apply to.
*/
if (encblkno[i].v != CGD_CIPHER_CBC_ENCBLKNO1) {
if (strcmp(sc->sc_cfuncs->cf_name, "aes-cbc") &&
strcmp(sc->sc_cfuncs->cf_name, "3des-cbc") &&
strcmp(sc->sc_cfuncs->cf_name, "blowfish-cbc")) {
log(LOG_WARNING, "cgd: %s only makes sense for cbc,"
" not for %s; ignoring\n",
encblkno[i].n, sc->sc_cfuncs->cf_name);
sc->sc_cdata.cf_mode = CGD_CIPHER_CBC_ENCBLKNO1;
} else {
log(LOG_WARNING, "cgd: enabling legacy encblkno8\n");
}
}

sc->sc_cdata.cf_keylen = ci->ci_keylen;
sc->sc_cdata.cf_priv = sc->sc_cfuncs->cf_init(ci->ci_keylen, inbuf,
&sc->sc_cdata.cf_blocksize);
if (sc->sc_cdata.cf_blocksize > CGD_MAXBLOCKSIZE) {
log(LOG_WARNING, "cgd: Disallowed cipher with blocksize %zu > %u\n",
sc->sc_cdata.cf_blocksize, CGD_MAXBLOCKSIZE);
sc->sc_cdata.cf_priv = NULL;
}

/*
* The blocksize is supposed to be in bytes. Unfortunately originally
* it was expressed in bits. For compatibility we maintain encblkno
* and encblkno8.
*/
sc->sc_cdata.cf_blocksize /= encblkno[i].d;
(void)explicit_memset(inbuf, 0, MAX_KEYSIZE);
if (!sc->sc_cdata.cf_priv) {
ret = EINVAL; /* XXX is this the right error? */
goto bail;
}
kmem_free(inbuf, MAX_KEYSIZE);

bufq_alloc(&dksc->sc_bufq, "fcfs", 0);

sc->sc_data = kmem_alloc(MAXPHYS, KM_SLEEP);
sc->sc_data_used = false;

/* Attach the disk. */
dk_attach(dksc);
disk_attach(&dksc->sc_dkdev);

disk_set_info(dksc->sc_dev, &dksc->sc_dkdev, NULL);

/* Discover wedges on this disk. */
dkwedge_discover(&dksc->sc_dkdev);

return 0;

bail:
kmem_free(inbuf, MAX_KEYSIZE);
(void)vn_close(vp, FREAD|FWRITE, l->l_cred);
return ret;
}

/* ARGSUSED */
static int
cgd_ioctl_clr(struct cgd_softc *sc, struct lwp *l)
{
struct dk_softc *dksc = &sc->sc_dksc;

if (!DK_ATTACHED(dksc))
return ENXIO;

/* Delete all of our wedges. */
dkwedge_delall(&dksc->sc_dkdev);

/* Kill off any queued buffers. */
dk_drain(dksc);
bufq_free(dksc->sc_bufq);

(void)vn_close(sc->sc_tvn, FREAD|FWRITE, l->l_cred);
sc->sc_cfuncs->cf_destroy(sc->sc_cdata.cf_priv);
kmem_free(sc->sc_tpath, sc->sc_tpathlen);
kmem_free(sc->sc_data, MAXPHYS);
sc->sc_data_used = false;
dk_detach(dksc);
disk_detach(&dksc->sc_dkdev);

return 0;
}

static int
cgd_ioctl_get(dev_t dev, void *data, struct lwp *l)
{
struct cgd_softc *sc;
struct cgd_user *cgu;
int unit, error;

unit = CGDUNIT(dev);
cgu = (struct cgd_user *)data;

DPRINTF_FOLLOW(("cgd_ioctl_get(0x%"PRIx64", %d, %p, %p)\n",
dev, unit, data, l));

/* XXX, we always return this units data, so if cgu_unit is
* not -1, that field doesn't match the rest
*/
if (cgu->cgu_unit == -1)
cgu->cgu_unit = unit;

if (cgu->cgu_unit < 0)
return EINVAL; /* XXX: should this be ENXIO? */

error = cgd_lock(false);
if (error)
return error;

sc = device_lookup_private(&cgd_cd, unit);
if (sc == NULL || !DK_ATTACHED(&sc->sc_dksc)) {
cgu->cgu_dev = 0;
cgu->cgu_alg[0] = '\0';
cgu->cgu_blocksize = 0;
cgu->cgu_mode = 0;
cgu->cgu_keylen = 0;
}
else {
mutex_enter(&sc->sc_lock);
cgu->cgu_dev = sc->sc_tdev;
strncpy(cgu->cgu_alg, sc->sc_cfuncs->cf_name,
sizeof(cgu->cgu_alg));
cgu->cgu_blocksize = sc->sc_cdata.cf_blocksize;
cgu->cgu_mode = sc->sc_cdata.cf_mode;
cgu->cgu_keylen = sc->sc_cdata.cf_keylen;
mutex_exit(&sc->sc_lock);
}

cgd_unlock();
return 0;
}

static int
cgdinit(struct cgd_softc *sc, const char *cpath, struct vnode *vp,
struct lwp *l)
{
struct disk_geom *dg;
int ret;
char *tmppath;
uint64_t psize;
unsigned secsize;
struct dk_softc *dksc = &sc->sc_dksc;

sc->sc_tvn = vp;
sc->sc_tpath = NULL;

tmppath = kmem_alloc(MAXPATHLEN, KM_SLEEP);
ret = copyinstr(cpath, tmppath, MAXPATHLEN, &sc->sc_tpathlen);
if (ret)
goto bail;
sc->sc_tpath = kmem_alloc(sc->sc_tpathlen, KM_SLEEP);
memcpy(sc->sc_tpath, tmppath, sc->sc_tpathlen);

sc->sc_tdev = vp->v_rdev;

if ((ret = getdisksize(vp, &psize, &secsize)) != 0)
goto bail;

if (psize == 0) {
ret = ENODEV;
goto bail;
}

/*
* XXX here we should probe the underlying device. If we
* are accessing a partition of type RAW_PART, then
* we should populate our initial geometry with the
* geometry that we discover from the device.
*/
dg = &dksc->sc_dkdev.dk_geom;
memset(dg, 0, sizeof(*dg));
dg->dg_secperunit = psize;
dg->dg_secsize = secsize;
dg->dg_ntracks = 1;
dg->dg_nsectors = 1024 * 1024 / dg->dg_secsize;
dg->dg_ncylinders = dg->dg_secperunit / dg->dg_nsectors;

bail:
kmem_free(tmppath, MAXPATHLEN);
if (ret && sc->sc_tpath)
kmem_free(sc->sc_tpath, sc->sc_tpathlen);
return ret;
}

/*
* Our generic cipher entry point. This takes care of the
* IV mode and passes off the work to the specific cipher.
* We implement here the IV method ``encrypted block
* number''.
*
* XXXrcd: for now we rely on our own crypto framework defined
* in dev/cgd_crypto.c. This will change when we
* get a generic kernel crypto framework.
*/

static void
blkno2blkno_buf(char *sbuf, daddr_t blkno)
{
int i;

/* Set up the blkno in blkno_buf, here we do not care much
* about the final layout of the information as long as we
* can guarantee that each sector will have a different IV
* and that the endianness of the machine will not affect
* the representation that we have chosen.
*
* We choose this representation, because it does not rely
* on the size of buf (which is the blocksize of the cipher),
* but allows daddr_t to grow without breaking existing
* disks.
*
* Note that blkno2blkno_buf does not take a size as input,
* and hence must be called on a pre-zeroed buffer of length
* greater than or equal to sizeof(daddr_t).
*/
for (i=0; i < sizeof(daddr_t); i++) {
*sbuf++ = blkno & 0xff;
blkno >>= 8;
}
}

static struct cpu_info *
cgd_cpu(struct cgd_softc *sc)
{
struct cgd_worker *cw = sc->sc_worker;
struct cpu_info *ci = NULL;
u_int cidx, i;

if (cw->cw_busy == 0) {
cw->cw_last = cpu_index(curcpu());
return NULL;
}

for (i=0, cidx = cw->cw_last+1; i<maxcpus; ++i, ++cidx) {
if (cidx >= maxcpus)
cidx = 0;
ci = cpu_lookup(cidx);
if (ci) {
cw->cw_last = cidx;
break;
}
}

return ci;
}

static void
cgd_enqueue(struct cgd_softc *sc, struct cgd_xfer *cx)
{
struct cgd_worker *cw = sc->sc_worker;
struct cpu_info *ci;

mutex_enter(&cw->cw_lock);
ci = cgd_cpu(sc);
cw->cw_busy++;
mutex_exit(&cw->cw_lock);

workqueue_enqueue(cw->cw_wq, &cx->cx_work, ci);
}

static void
cgd_process(struct work *wk, void *arg)
{
struct cgd_xfer *cx = (struct cgd_xfer *)wk;
struct cgd_softc *sc = cx->cx_sc;
struct cgd_worker *cw = sc->sc_worker;

cgd_cipher(sc, cx->cx_dstv, cx->cx_srcv, cx->cx_len,
cx->cx_blkno, cx->cx_secsize, cx->cx_dir);

if (cx->cx_dir == CGD_CIPHER_ENCRYPT) {
cgd_diskstart2(sc, cx);
} else {
cgd_iodone2(sc, cx);
}

mutex_enter(&cw->cw_lock);
if (cw->cw_busy > 0)
cw->cw_busy--;
mutex_exit(&cw->cw_lock);
}

static void
cgd_cipher(struct cgd_softc *sc, void *dstv, const void *srcv,
size_t len, daddr_t blkno, size_t secsize, int dir)
{
char *dst = dstv;
const char *src = srcv;
cfunc_cipher *cipher = sc->sc_cfuncs->cf_cipher;
size_t blocksize = sc->sc_cdata.cf_blocksize;
size_t todo;
char blkno_buf[CGD_MAXBLOCKSIZE] __aligned(CGD_BLOCKALIGN);

DPRINTF_FOLLOW(("cgd_cipher() dir=%d\n", dir));

if (sc->sc_cdata.cf_mode == CGD_CIPHER_CBC_ENCBLKNO8)
blocksize /= 8;

KASSERT(len % blocksize == 0);
/* ensure that sizeof(daddr_t) <= blocksize (for encblkno IVing) */
KASSERT(sizeof(daddr_t) <= blocksize);
KASSERT(blocksize <= CGD_MAXBLOCKSIZE);

for (; len > 0; len -= todo) {
todo = MIN(len, secsize);

memset(blkno_buf, 0x0, blocksize);
blkno2blkno_buf(blkno_buf, blkno);
IFDEBUG(CGDB_CRYPTO, hexprint("step 1: blkno_buf",
blkno_buf, blocksize));

/*
* Handle bollocksed up encblkno8 mistake. We used to
* compute the encryption of a zero block with blkno as
* the CBC IV -- except in an early mistake arising
* from bit/byte confusion, we actually computed the
* encryption of the last of _eight_ zero blocks under
* CBC as the CBC IV.
*
* Encrypting the block number is handled inside the
* cipher dispatch now (even though in practice, both
* CBC and XTS will do the same thing), so we have to
* simulate the block number that would yield the same
* result. So we encrypt _six_ zero blocks -- the
* first one and the last one are handled inside the
* cipher dispatch.
*/
if (sc->sc_cdata.cf_mode == CGD_CIPHER_CBC_ENCBLKNO8) {
static const uint8_t zero[CGD_MAXBLOCKSIZE];
uint8_t iv[CGD_MAXBLOCKSIZE];

memcpy(iv, blkno_buf, blocksize);
cipher(sc->sc_cdata.cf_priv, blkno_buf, zero,
6*blocksize, iv, CGD_CIPHER_ENCRYPT);
memmove(blkno_buf, blkno_buf + 5*blocksize, blocksize);
}

cipher(sc->sc_cdata.cf_priv, dst, src, todo, blkno_buf, dir);

dst += todo;
src += todo;
blkno++;
}
}

#ifdef DEBUG
static void
hexprint(const char *start, void *buf, int len)
{
char *c = buf;

KASSERTMSG(len >= 0, "hexprint: called with len < 0");
printf("%s: len=%06d 0x", start, len);
while (len--)
printf("%02x", (unsigned char) *c++);
}
#endif

static void
cgd_selftest(void)
{
struct cgd_softc sc;
void *buf;

for (size_t i = 0; i < __arraycount(selftests); i++) {
const char *alg = selftests[i].alg;
int encblkno8 = selftests[i].encblkno8;
const uint8_t *key = selftests[i].key;
int keylen = selftests[i].keylen;
int txtlen = selftests[i].txtlen;

aprint_debug("cgd: self-test %s-%d%s\n", alg, keylen,
encblkno8 ? " (encblkno8)" : "");

memset(&sc, 0, sizeof(sc));

sc.sc_cfuncs = cryptfuncs_find(alg);
if (sc.sc_cfuncs == NULL)
panic("%s not implemented", alg);

sc.sc_cdata.cf_blocksize = 8 * selftests[i].blocksize;
sc.sc_cdata.cf_mode = encblkno8 ? CGD_CIPHER_CBC_ENCBLKNO8 :
CGD_CIPHER_CBC_ENCBLKNO1;
sc.sc_cdata.cf_keylen = keylen;

sc.sc_cdata.cf_priv = sc.sc_cfuncs->cf_init(keylen,
key, &sc.sc_cdata.cf_blocksize);
if (sc.sc_cdata.cf_priv == NULL)
panic("cf_priv is NULL");
if (sc.sc_cdata.cf_blocksize > CGD_MAXBLOCKSIZE)
panic("bad block size %zu", sc.sc_cdata.cf_blocksize);

if (!encblkno8)
sc.sc_cdata.cf_blocksize /= 8;

buf = kmem_alloc(txtlen, KM_SLEEP);
memcpy(buf, selftests[i].ptxt, txtlen);

cgd_cipher(&sc, buf, buf, txtlen, selftests[i].blkno,
selftests[i].secsize, CGD_CIPHER_ENCRYPT);
if (memcmp(buf, selftests[i].ctxt, txtlen) != 0) {
hexdump(printf, "was", buf, txtlen);
hexdump(printf, "exp", selftests[i].ctxt, txtlen);
panic("cgd %s-%d encryption is broken [%zu]",
selftests[i].alg, keylen, i);
}

cgd_cipher(&sc, buf, buf, txtlen, selftests[i].blkno,
selftests[i].secsize, CGD_CIPHER_DECRYPT);
if (memcmp(buf, selftests[i].ptxt, txtlen) != 0) {
hexdump(printf, "was", buf, txtlen);
hexdump(printf, "exp", selftests[i].ptxt, txtlen);
panic("cgd %s-%d decryption is broken [%zu]",
selftests[i].alg, keylen, i);
}

kmem_free(buf, txtlen);
sc.sc_cfuncs->cf_destroy(sc.sc_cdata.cf_priv);
}

aprint_debug("cgd: self-tests passed\n");
}

MODULE(MODULE_CLASS_DRIVER, cgd, "blowfish,des,dk_subr,bufq_fcfs");

#ifdef _MODULE
CFDRIVER_DECL(cgd, DV_DISK, NULL);

devmajor_t cgd_bmajor = -1, cgd_cmajor = -1;
#endif

static int
cgd_modcmd(modcmd_t cmd, void *arg)
{
int error = 0;

switch (cmd) {
case MODULE_CMD_INIT:
#ifdef _MODULE
mutex_init(&cgd_spawning_mtx, MUTEX_DEFAULT, IPL_NONE);
cv_init(&cgd_spawning_cv, "cgspwn");

/*
* Attach the {b,c}devsw's
*/
error = devsw_attach("cgd", &cgd_bdevsw, &cgd_bmajor,
&cgd_cdevsw, &cgd_cmajor);
if (error) {
aprint_error("%s: unable to attach %s devsw, "
"error %d", __func__, cgd_cd.cd_name, error);
break;
}

/*
* Attach to autoconf database
*/
error = config_cfdriver_attach(&cgd_cd);
if (error) {
devsw_detach(&cgd_bdevsw, &cgd_cdevsw);
aprint_error("%s: unable to register cfdriver for"
"%s, error %d\n", __func__, cgd_cd.cd_name, error);
break;
}

error = config_cfattach_attach(cgd_cd.cd_name, &cgd_ca);
if (error) {
config_cfdriver_detach(&cgd_cd);
devsw_detach(&cgd_bdevsw, &cgd_cdevsw);
aprint_error("%s: unable to register cfattach for"
"%s, error %d\n", __func__, cgd_cd.cd_name, error);
break;
}
#endif
break;

case MODULE_CMD_FINI:
#ifdef _MODULE
/*
* Remove device from autoconf database
*/
error = config_cfattach_detach(cgd_cd.cd_name, &cgd_ca);
if (error) {
aprint_error("%s: failed to detach %s cfattach, "
"error %d\n", __func__, cgd_cd.cd_name, error);
break;
}
error = config_cfdriver_detach(&cgd_cd);
if (error) {
(void)config_cfattach_attach(cgd_cd.cd_name, &cgd_ca);
aprint_error("%s: failed to detach %s cfdriver, "
"error %d\n", __func__, cgd_cd.cd_name, error);
break;
}

/*
* Remove {b,c}devsw's
*/
devsw_detach(&cgd_bdevsw, &cgd_cdevsw);

cv_destroy(&cgd_spawning_cv);
mutex_destroy(&cgd_spawning_mtx);
#endif
break;

case MODULE_CMD_STAT:
error = ENOTTY;
break;
default:
error = ENOTTY;
break;
}

return error;
}