/*      $NetBSD: apei_einj.c,v 1.7 2024/03/28 13:40:08 riastradh Exp $  */

/*-
* Copyright (c) 2024 The NetBSD Foundation, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
*    notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
*    notice, this list of conditions and the following disclaimer in the
*    documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/

/*
* APEI EINJ -- Error Injection Table
*
* https://uefi.org/specs/ACPI/6.5/18_Platform_Error_Interfaces.html#error-injection
*
* XXX Consider a /dev node with ioctls for error injection rather than
* the somewhat kooky sysctl interface.  By representing an error
* injection request in a structure, we can serialize access to the
* platform's EINJ operational context.  However, this also requires
* some nontrivial userland support; maybe relying on the user to tread
* carefully with error injection is fine -- after all, many types of
* error injection will cause a system halt/panic.
*
* XXX Properly expose SET_ERROR_TYPE_WITH_ADDRESS, which has a more
* complicated relationship with its RegisterRegion field.
*/

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: apei_einj.c,v 1.7 2024/03/28 13:40:08 riastradh Exp $");

#include <sys/types.h>

#include <sys/device.h>
#include <sys/sysctl.h>
#include <sys/systm.h>

#include <dev/acpi/acpivar.h>
#include <dev/acpi/apei_einjvar.h>
#include <dev/acpi/apei_interp.h>
#include <dev/acpi/apei_mapreg.h>
#include <dev/acpi/apei_reg.h>
#include <dev/acpi/apeivar.h>

#include "ioconf.h"

#define _COMPONENT      ACPI_RESOURCE_COMPONENT
ACPI_MODULE_NAME        ("apei")

static void apei_einj_instfunc(ACPI_WHEA_HEADER *, struct apei_mapreg *,
   void *, uint32_t *, uint32_t);
static uint64_t apei_einj_act(struct apei_softc *, enum AcpiEinjActions,
   uint64_t);
static uint64_t apei_einj_trigger(struct apei_softc *, uint64_t);
static int apei_einj_action_sysctl(SYSCTLFN_ARGS);
static int apei_einj_trigger_sysctl(SYSCTLFN_ARGS);
static int apei_einj_types_sysctl(SYSCTLFN_ARGS);

/*
* apei_einj_action
*
*      Symbolic names of the APEI EINJ (Error Injection) logical actions
*      are taken (and downcased) from:
*
*      https://uefi.org/specs/ACPI/6.5/18_Platform_Error_Interfaces.html#error-injection-actions
*/
static const char *const apei_einj_action[] = {
       [ACPI_EINJ_BEGIN_OPERATION] = "begin_injection_operation",
       [ACPI_EINJ_GET_TRIGGER_TABLE] = "get_trigger_error_action_table",
       [ACPI_EINJ_SET_ERROR_TYPE] = "set_error_type",
       [ACPI_EINJ_GET_ERROR_TYPE] = "get_error_type",
       [ACPI_EINJ_END_OPERATION] = "end_operation",
       [ACPI_EINJ_EXECUTE_OPERATION] = "execute_operation",
       [ACPI_EINJ_CHECK_BUSY_STATUS] = "check_busy_status",
       [ACPI_EINJ_GET_COMMAND_STATUS] = "get_command_status",
       [ACPI_EINJ_SET_ERROR_TYPE_WITH_ADDRESS] = "set_error_type_with_address",
       [ACPI_EINJ_GET_EXECUTE_TIMINGS] = "get_execute_operation_timings",
};

/*
* apei_einj_instruction
*
*      Symbolic names of the APEI EINJ (Error Injection) instructions to
*      implement logical actions are taken (and downcased) from:
*
*      https://uefi.org/specs/ACPI/6.5/18_Platform_Error_Interfaces.html#injection-instructions-table
*/

static const char *const apei_einj_instruction[] = {
       [ACPI_EINJ_READ_REGISTER] = "read_register",
       [ACPI_EINJ_READ_REGISTER_VALUE] = "read_register",
       [ACPI_EINJ_WRITE_REGISTER] = "write_register",
       [ACPI_EINJ_WRITE_REGISTER_VALUE] = "write_register_value",
       [ACPI_EINJ_NOOP] = "noop",
};

/*
* apei_einj_instreg
*
*      Table of which instructions use a register operand.
*
*      Must match apei_einj_instfunc.
*/
static const bool apei_einj_instreg[] = {
       [ACPI_EINJ_READ_REGISTER] = true,
       [ACPI_EINJ_READ_REGISTER_VALUE] = true,
       [ACPI_EINJ_WRITE_REGISTER] = true,
       [ACPI_EINJ_WRITE_REGISTER_VALUE] = true,
       [ACPI_EINJ_NOOP] = false,
};

/*
* apei_einj_attach(sc)
*
*      Scan the Error Injection table to ascertain what error
*      injection actions the firmware supports and how to perform
*      them.  Create sysctl nodes for triggering error injection.
*/
void
apei_einj_attach(struct apei_softc *sc)
{
       ACPI_TABLE_EINJ *einj = sc->sc_tab.einj;
       struct apei_einj_softc *jsc = &sc->sc_einj;
       ACPI_EINJ_ENTRY *entry;
       const struct sysctlnode *sysctl_einj;
       const struct sysctlnode *sysctl_einj_action;
       uint32_t i, nentries, maxnentries;
       unsigned action;
       int error;

       /*
        * Verify the table length, table header length, and
        * instruction entry count are all sensible.  If the header is
        * truncated, stop here; if the entries are truncated, stop at
        * the largest integral number of full entries that fits.
        */
       if (einj->Header.Length < sizeof(*einj)) {
               aprint_error_dev(sc->sc_dev, "EINJ: truncated table:"
                   " %"PRIu32" < %zu minimum bytes\n",
                   einj->Header.Length, sizeof(*einj));
               return;
       }
       if (einj->HeaderLength <
           sizeof(*einj) - offsetof(ACPI_TABLE_EINJ, HeaderLength)) {
               aprint_error_dev(sc->sc_dev, "EINJ: truncated header:"
                   " %"PRIu32" < %zu bytes\n",
                   einj->HeaderLength,
                   sizeof(*einj) - offsetof(ACPI_TABLE_EINJ, HeaderLength));
               return;
       }
       nentries = einj->Entries;
       maxnentries = (einj->Header.Length - sizeof(*einj))/sizeof(*entry);
       if (nentries > maxnentries) {
               aprint_error_dev(sc->sc_dev, "EINJ: excessive entries:"
                   " %"PRIu32", truncating to %"PRIu32"\n",
                   nentries, maxnentries);
               nentries = maxnentries;
       }
       if (nentries*sizeof(*entry) < einj->Header.Length - sizeof(*einj)) {
               aprint_error_dev(sc->sc_dev, "EINJ:"
                   " %zu bytes of trailing garbage after last entry\n",
                   einj->Header.Length - nentries*sizeof(*entry));
       }

       /*
        * Create sysctl hw.acpi.apei.einj for all EINJ-related knobs.
        */
       error = sysctl_createv(&sc->sc_sysctllog, 0,
           &sc->sc_sysctlroot, &sysctl_einj, 0,
           CTLTYPE_NODE, "einj",
           SYSCTL_DESCR("Error injection"),
           NULL, 0, NULL, 0,
           CTL_CREATE, CTL_EOL);
       if (error) {
               aprint_error_dev(sc->sc_dev, "failed to create"
                   " hw.acpi.apei.einj: %d\n", error);
               sysctl_einj = NULL;
       }

       /*
        * Create an interpreter for EINJ actions.
        */
       jsc->jsc_interp = apei_interp_create("EINJ",
           apei_einj_action, __arraycount(apei_einj_action),
           apei_einj_instruction, __arraycount(apei_einj_instruction),
           apei_einj_instreg, /*instvalid*/NULL, apei_einj_instfunc);

       /*
        * Compile the interpreter from the EINJ action instruction
        * table.
        */
       entry = (ACPI_EINJ_ENTRY *)(einj + 1);
       for (i = 0; i < nentries; i++, entry++)
               apei_interp_pass1_load(jsc->jsc_interp, i, &entry->WheaHeader);
       entry = (ACPI_EINJ_ENTRY *)(einj + 1);
       for (i = 0; i < nentries; i++, entry++) {
               apei_interp_pass2_verify(jsc->jsc_interp, i,
                   &entry->WheaHeader);
       }
       apei_interp_pass3_alloc(jsc->jsc_interp);
       entry = (ACPI_EINJ_ENTRY *)(einj + 1);
       for (i = 0; i < nentries; i++, entry++) {
               apei_interp_pass4_assemble(jsc->jsc_interp, i,
                   &entry->WheaHeader);
       }
       apei_interp_pass5_verify(jsc->jsc_interp);

       /*
        * Create sysctl hw.acpi.apei.einj.action for individual actions.
        */
       error = sysctl_einj == NULL ? ENOENT :
           sysctl_createv(&sc->sc_sysctllog, 0,
               &sysctl_einj, &sysctl_einj_action, 0,
               CTLTYPE_NODE, "action",
               SYSCTL_DESCR("EINJ actions"),
               NULL, 0, NULL, 0,
               CTL_CREATE, CTL_EOL);
       if (error) {
               aprint_error_dev(sc->sc_dev, "failed to create"
                   " hw.acpi.apei.einj.action: %d\n", error);
               sysctl_einj_action = NULL;
       }

       /*
        * Create sysctl nodes for each action we know about.
        */
       for (action = 0; action < __arraycount(apei_einj_action); action++) {
               if (apei_einj_action[action] == NULL)
                       continue;

               /*
                * Check to see if there are any instructions for this
                * action.
                *
                * XXX Maybe add this to the apei_interp.h abstraction.
                */
               entry = (ACPI_EINJ_ENTRY *)(einj + 1);
               for (i = 0; i < nentries; i++, entry++) {
                       ACPI_WHEA_HEADER *const header = &entry->WheaHeader;

                       if (action == header->Action)
                               break;
               }
               if (i == nentries) {
                       /*
                        * No instructions for this action, so assume
                        * it's not supported.
                        */
                       continue;
               }

               /*
                * Create a sysctl knob to perform the action.
                */
               error = sysctl_einj_action == NULL ? ENOENT :
                   sysctl_createv(&sc->sc_sysctllog, 0,
                       &sysctl_einj_action, NULL, CTLFLAG_READWRITE,
                       CTLTYPE_QUAD, apei_einj_action[action],
                       NULL,   /* description */
                       &apei_einj_action_sysctl, 0, NULL, 0,
                       action, CTL_EOL);
               if (error) {
                       aprint_error_dev(sc->sc_dev, "failed to create"
                           " sysctl hw.acpi.apei.einj.action.%s: %d\n",
                           apei_einj_action[action], error);
                       continue;
               }
       }

       /*
        * Create a sysctl knob to trigger error.
        */
       error = sysctl_einj == NULL ? ENOENT :
           sysctl_createv(&sc->sc_sysctllog, 0,
               &sysctl_einj, NULL, CTLFLAG_READWRITE,
               CTLTYPE_QUAD, "trigger",
               NULL,   /* description */
               &apei_einj_trigger_sysctl, 0, NULL, 0,
               CTL_CREATE, CTL_EOL);
       if (error) {
               aprint_error_dev(sc->sc_dev, "failed to create"
                   " sysctl hw.acpi.apei.einj.trigger: %d\n",
                   error);
       }

       /*
        * Query the available types of error to inject and print it to
        * dmesg.
        *
        * https://uefi.org/specs/ACPI/6.5/18_Platform_Error_Interfaces.html#error-types
        */
       uint64_t types = apei_einj_act(sc, ACPI_EINJ_GET_ERROR_TYPE, 0);
       char typesbuf[1024], *typesp;
       /* XXX define this format somewhere */
       snprintb_m(typesbuf, sizeof(typesbuf), "\177\020"
           "b\000"     "PROC_CORRECTABLE\0"
           "b\001"     "PROC_UNCORRECTABLE\0"
           "b\002"     "PROC_FATAL\0"
           "b\003"     "MEM_CORRECTABLE\0"
           "b\004"     "MEM_UNCORRECTABLE\0"
           "b\005"     "MEM_FATAL\0"
           "b\006"     "PCIE_CORRECTABLE\0"
           "b\007"     "PCIE_UNCORRECTABLE\0"
           "b\010"     "PCIE_FATAL\0"
           "b\011"     "PLAT_CORRECTABLE\0"
           "b\012"     "PLAT_UNCORRECTABLE\0"
           "b\013"     "PLAT_FATAL\0"
           "b\014"     "CXLCACHE_CORRECTABLE\0"
           "b\015"     "CXLCACHE_UNCORRECTABLE\0"
           "b\016"     "CXLCACHE_FATAL\0"
           "b\017"     "CXLMEM_CORRECTABLE\0"
           "b\020"     "CXLMEM_UNCORRECTABLE\0"
           "b\021"     "CXLMEM_FATAL\0"
//          "f\022\014" "reserved\0"
           "b\036"     "EINJv2\0"
           "b\037"     "VENDOR\0"
           "\0", types, 36);
       for (typesp = typesbuf; strlen(typesp); typesp += strlen(typesp) + 1) {
               aprint_normal_dev(sc->sc_dev, "EINJ: can inject:"
                   " %s\n", typesp);
       }

       /*
        * Create a sysctl knob to query the available types of error
        * to inject.  In principle this could change dynamically, so
        * we'll make it dynamic.
        */
       error = sysctl_einj == NULL ? ENOENT :
           sysctl_createv(&sc->sc_sysctllog, 0,
               &sysctl_einj, NULL, 0,
               CTLTYPE_QUAD, "types",
               SYSCTL_DESCR("Types of errors that can be injected"),
               &apei_einj_types_sysctl, 0, NULL, 0,
               CTL_CREATE, CTL_EOL);
       if (error) {
               aprint_error_dev(sc->sc_dev, "failed to create"
                   " sysctl hw.acpi.apei.einj.types: %d\n",
                   error);
       }
}

/*
* apei_einj_detach(sc)
*
*      Free any software resources associated with the Error Injection
*      table.
*/
void
apei_einj_detach(struct apei_softc *sc)
{
       struct apei_einj_softc *jsc = &sc->sc_einj;

       if (jsc->jsc_interp) {
               apei_interp_destroy(jsc->jsc_interp);
               jsc->jsc_interp = NULL;
       }
}

/*
* struct apei_einj_machine
*
*      Machine state for executing EINJ instructions.
*/
struct apei_einj_machine {
       struct apei_softc       *sc;
       uint64_t                x;      /* in */
       uint64_t                y;      /* out */
};

/*
* apei_einj_instfunc(header, cookie, &ip, maxip)
*
*      Run a single instruction in the service of performing an EINJ
*      action.  Updates the EINJ machine at cookie in place.
*
*      This doesn't read or write ip.  The TRIGGER_ERROR logic relies
*      on this; if you change the fact, you must update that logic
*      too.
*/
static void
apei_einj_instfunc(ACPI_WHEA_HEADER *header, struct apei_mapreg *map,
   void *cookie, uint32_t *ipp, uint32_t maxip)
{
       struct apei_einj_machine *M = cookie;

       /*
        * Abbreviate some of the intermediate quantities to make the
        * instruction logic conciser and more legible.
        */
       const uint8_t BitOffset = header->RegisterRegion.BitOffset;
       const uint64_t Mask = header->Mask;
       const uint64_t Value = header->Value;
       ACPI_GENERIC_ADDRESS *const reg = &header->RegisterRegion;
       const bool preserve_register = header->Flags & ACPI_EINJ_PRESERVE;

       aprint_debug_dev(M->sc->sc_dev, "%s: instr=0x%02"PRIx8
           " (%s)"
           " Address=0x%"PRIx64
           " BitOffset=%"PRIu8" Mask=0x%"PRIx64" Value=0x%"PRIx64
           " Flags=0x%"PRIx8"\n",
           __func__, header->Instruction,
           (header->Instruction < __arraycount(apei_einj_instruction)
               ? apei_einj_instruction[header->Instruction]
               : "unknown"),
           reg->Address,
           BitOffset, Mask, Value,
           header->Flags);

       /*
        * Zero-initialize the output by default.
        */
       M->y = 0;

       /*
        * Dispatch the instruction.
        */
       switch (header->Instruction) {
       case ACPI_EINJ_READ_REGISTER:
               M->y = apei_read_register(reg, map, Mask);
               break;
       case ACPI_EINJ_READ_REGISTER_VALUE: {
               uint64_t v;

               v = apei_read_register(reg, map, Mask);
               M->y = (v == Value ? 1 : 0);
               break;
       }
       case ACPI_EINJ_WRITE_REGISTER:
               apei_write_register(reg, map, Mask, preserve_register, M->x);
               break;
       case ACPI_EINJ_WRITE_REGISTER_VALUE:
               apei_write_register(reg, map, Mask, preserve_register, Value);
               break;
       case ACPI_EINJ_NOOP:
               break;
       default:                /* XXX unreachable */
               break;
       }
}

/*
* apei_einj_act(sc, action, x)
*
*      Perform the named EINJ action with input x, by executing the
*      instruction defined for the action by the EINJ, and return the
*      output.
*/
static uint64_t
apei_einj_act(struct apei_softc *sc, enum AcpiEinjActions action,
   uint64_t x)
{
       struct apei_einj_softc *const jsc = &sc->sc_einj;
       struct apei_einj_machine einj_machine, *const M = &einj_machine;

       aprint_debug_dev(sc->sc_dev, "%s: action=%d (%s) input=0x%"PRIx64"\n",
           __func__,
           action,
           (action < __arraycount(apei_einj_action)
               ? apei_einj_action[action]
               : "unknown"),
           x);

       /*
        * Initialize the machine to execute the action's instructions.
        */
       memset(M, 0, sizeof(*M));
       M->sc = sc;
       M->x = x;               /* input */
       M->y = 0;               /* output */

       /*
        * Run the interpreter.
        */
       apei_interpret(jsc->jsc_interp, action, M);

       /*
        * Return the result.
        */
       aprint_debug_dev(sc->sc_dev, "%s: output=0x%"PRIx64"\n", __func__,
           M->y);
       return M->y;
}

/*
* apei_einj_trigger(sc, x)
*
*      Obtain the TRIGGER_ERROR action table and, if there is anything
*      to be done with it, execute it with input x and return the
*      output.  If nothing is to be done, return 0.
*/
static uint64_t
apei_einj_trigger(struct apei_softc *sc, uint64_t x)
{
       uint64_t teatab_pa;
       ACPI_EINJ_TRIGGER *teatab = NULL;
       size_t mapsize = 0, tabsize, bodysize;
       ACPI_EINJ_ENTRY *entry;
       struct apei_einj_machine einj_machine, *const M = &einj_machine;
       uint32_t i, nentries;

       /*
        * Initialize the machine to execute the TRIGGER_ERROR action's
        * instructions.  Do this early to keep the error branches
        * simpler.
        */
       memset(M, 0, sizeof(*M));
       M->sc = sc;
       M->x = x;               /* input */
       M->y = 0;               /* output */

       /*
        * Get the TRIGGER_ERROR action table's physical address.
        */
       teatab_pa = apei_einj_act(sc, ACPI_EINJ_GET_TRIGGER_TABLE, 0);

       /*
        * Map just the header.  We don't know how large the table is
        * because we get that from the header.
        */
       mapsize = sizeof(*teatab);
       teatab = AcpiOsMapMemory(teatab_pa, mapsize);

       /*
        * If there's no entries, stop here -- nothing to do separately
        * to trigger an error report.
        */
       nentries = teatab->EntryCount;
       if (nentries == 0)
               goto out;

       /*
        * If the header size or the table size is nonsense, bail.
        */
       if (teatab->HeaderSize < sizeof(*teatab) ||
           teatab->TableSize < teatab->HeaderSize) {
               device_printf(sc->sc_dev, "TRIGGER_ERROR action table:"
                   " invalid sizes:"
                   " HeaderSize=%"PRIu32" TableSize=%"PRIu32"\n",
                   teatab->HeaderSize, teatab->TableSize);
       }

       /*
        * If the revision is nonzero, we don't know what to do.  I've
        * only seen revision zero so far, and the spec doesn't say
        * anything about revisions that I've found.
        */
       if (teatab->Revision != 0) {
               device_printf(sc->sc_dev, "TRIGGER_ERROR action table:"
                   " unknown revision: %"PRIx32"\n", teatab->Revision);
               goto out;
       }

       /*
        * Truncate the table to the number of entries requested and
        * ignore trailing garbage if the table is long, or round the
        * number of entries down to what fits in the table if the
        * table is short.
        */
       tabsize = teatab->TableSize;
       bodysize = tabsize - teatab->HeaderSize;
       if (nentries < howmany(bodysize, sizeof(ACPI_EINJ_ENTRY))) {
               device_printf(sc->sc_dev, "TRIGGER_ERROR action table:"
                   " %zu bytes of trailing garbage\n",
                   tabsize - nentries*sizeof(ACPI_EINJ_ENTRY));
               bodysize = nentries*sizeof(ACPI_EINJ_ENTRY);
               tabsize = teatab->HeaderSize + bodysize;
       } else if (nentries > howmany(bodysize, sizeof(ACPI_EINJ_ENTRY))) {
               device_printf(sc->sc_dev, "TRIGGER_ERROR action table:"
                   " truncated to %zu entries\n",
                   nentries*sizeof(ACPI_EINJ_ENTRY));
               nentries = howmany(bodysize, sizeof(ACPI_EINJ_ENTRY));
               bodysize = nentries*sizeof(ACPI_EINJ_ENTRY);
               tabsize = teatab->HeaderSize + bodysize;
       }

       /*
        * Unmap the header and map the whole table instead.
        */
       AcpiOsUnmapMemory(teatab, mapsize);
       mapsize = tabsize;
       teatab = AcpiOsMapMemory(teatab_pa, mapsize);

       /*
        * Now iterate over the EINJ-type entries and execute the
        * trigger error action instructions -- but skip if they're not
        * for the TRIGGER_ERROR action, and stop if they're truncated.
        *
        * Entries are fixed-size, so we can just index them.
        */
       entry = (ACPI_EINJ_ENTRY *)((char *)teatab + teatab->HeaderSize);
       for (i = 0; i < nentries; i++) {
               ACPI_WHEA_HEADER *const header = &entry[i].WheaHeader;

               /*
                * Verify the action is TRIGGER_ERROR.  If not, skip.
                */
               if (header->Action != ACPI_EINJ_TRIGGER_ERROR) {
                       device_printf(sc->sc_dev, "TRIGGER_ERROR action table:"
                           " other action: %"PRIu32" (%s)\n",
                           header->Action,
                           (header->Action < __arraycount(apei_einj_action)
                               ? apei_einj_action[header->Action]
                               : "unknown"));
                       continue;
               }

               /*
                * Verify the instruction.
                */
               if (header->Instruction >= __arraycount(apei_einj_instreg)) {
                       device_printf(sc->sc_dev, "TRIGGER_ERROR action table:"
                           " unknown instruction: %"PRIu32"\n",
                           header->Instruction);
                       continue;
               }

               /*
                * Map the register if needed.
                */
               struct apei_mapreg *map = NULL;
               if (apei_einj_instreg[header->Instruction]) {
                       map = apei_mapreg_map(&header->RegisterRegion);
                       if (map == NULL) {
                               device_printf(sc->sc_dev, "TRIGGER_ERROR"
                                   " action table: failed to map register\n");
                               continue;
                       }
               }

               /*
                * Execute the instruction.  Since there's only one
                * action, we don't bother with the apei_interp
                * machinery to collate instruction tables for each
                * action.  EINJ instructions don't change ip.
                */
               uint32_t ip = i + 1;
               apei_einj_instfunc(header, map, M, &ip, nentries);
               KASSERT(ip == i + 1);

               /*
                * Unmap the register if mapped.
                */
               if (map != NULL)
                       apei_mapreg_unmap(&header->RegisterRegion, map);
       }

out:    if (teatab) {
               AcpiOsUnmapMemory(teatab, mapsize);
               teatab = NULL;
               mapsize = 0;
       }
       return M->y;
}

/*
* apei_einj_action_sysctl:
*
*      Handle sysctl queries under hw.acpi.apei.einj.action.*.
*/
static int
apei_einj_action_sysctl(SYSCTLFN_ARGS)
{
       device_t apei0 = NULL;
       struct apei_softc *sc;
       enum AcpiEinjActions action;
       struct sysctlnode node = *rnode;
       uint64_t v;
       int error;

       /*
        * As a defence against mistakes, require the user to specify a
        * write.
        */
       if (newp == NULL) {
               error = ENOENT;
               goto out;
       }

       /*
        * Take a reference to the apei0 device so it doesn't go away
        * while we're working, and get the softc.
        */
       if ((apei0 = device_lookup_acquire(&apei_cd, 0)) == NULL) {
               error = ENXIO;
               goto out;
       }
       sc = device_private(apei0);

       /*
        * Fail if there's no EINJ.
        */
       if (sc->sc_tab.einj == NULL) {
               error = ENODEV;
               goto out;
       }

       /*
        * Identify the requested action.  If we don't recognize it,
        * fail with EINVAL.
        */
       switch (node.sysctl_num) {
       case ACPI_EINJ_BEGIN_OPERATION:
       case ACPI_EINJ_GET_TRIGGER_TABLE:
       case ACPI_EINJ_SET_ERROR_TYPE:
       case ACPI_EINJ_GET_ERROR_TYPE:
       case ACPI_EINJ_END_OPERATION:
       case ACPI_EINJ_EXECUTE_OPERATION:
       case ACPI_EINJ_CHECK_BUSY_STATUS:
       case ACPI_EINJ_GET_COMMAND_STATUS:
       case ACPI_EINJ_SET_ERROR_TYPE_WITH_ADDRESS:
       case ACPI_EINJ_GET_EXECUTE_TIMINGS:
               action = node.sysctl_num;
               break;
       default:
               error = ENOENT;
               goto out;
       }

       /*
        * Kludge: Copy the `new value' for the sysctl in as an input
        * to the injection action.
        */
       error = sysctl_copyin(curlwp, newp, &v, sizeof(v));
       if (error)
               goto out;

       /*
        * Perform the EINJ action by following the table's
        * instructions.
        */
       v = apei_einj_act(sc, action, v);

       /*
        * Return the output of the operation as the `old value' of the
        * sysctl.  This also updates v with what was written to the
        * sysctl was written, but we don't care because we already
        * read that in and acted on it.
        */
       node.sysctl_data = &v;
       error = sysctl_lookup(SYSCTLFN_CALL(&node));

out:    if (apei0) {
               device_release(apei0);
               apei0 = NULL;
       }
       return error;
}

/*
* apei_einj_trigger_sysctl
*
*      Handle sysctl hw.acpi.apei.einj.trigger.
*/
static int
apei_einj_trigger_sysctl(SYSCTLFN_ARGS)
{
       device_t apei0 = NULL;
       struct apei_softc *sc;
       struct sysctlnode node = *rnode;
       uint64_t v;
       int error;

       /*
        * As a defence against mistakes, require the user to specify a
        * write.
        */
       if (newp == NULL) {
               error = ENOENT;
               goto out;
       }

       /*
        * Take a reference to the apei0 device so it doesn't go away
        * while we're working, and get the softc.
        */
       if ((apei0 = device_lookup_acquire(&apei_cd, 0)) == NULL) {
               error = ENXIO;
               goto out;
       }
       sc = device_private(apei0);

       /*
        * Fail if there's no EINJ.
        */
       if (sc->sc_tab.einj == NULL) {
               error = ENODEV;
               goto out;
       }

       /*
        * Kludge: Copy the `new value' for the sysctl in as an input
        * to the trigger action.
        */
       error = sysctl_copyin(curlwp, newp, &v, sizeof(v));
       if (error)
               goto out;

       /*
        * Perform the TRIGGER_ERROR action.
        */
       v = apei_einj_trigger(sc, v);

       /*
        * Return the output of the operation as the `old value' of the
        * sysctl.  This also updates v with what was written to the
        * sysctl was written, but we don't care because we already
        * read that in and acted on it.
        */
       node.sysctl_data = &v;
       error = sysctl_lookup(SYSCTLFN_CALL(&node));

out:    if (apei0) {
               device_release(apei0);
               apei0 = NULL;
       }
       return error;
}

/*
* apei_einj_types_sysctl
*
*      Handle sysctl hw.acpi.apei.einj.types.
*/
static int
apei_einj_types_sysctl(SYSCTLFN_ARGS)
{
       device_t apei0 = NULL;
       struct apei_softc *sc;
       struct sysctlnode node = *rnode;
       uint64_t types;
       int error;

       /*
        * Take a reference to the apei0 device so it doesn't go away
        * while we're working, and get the softc.
        *
        * XXX Is this necessary?  Shouldn't sysctl_teardown take care
        * of preventing new sysctl calls and waiting until all pending
        * sysctl calls are done?
        */
       if ((apei0 = device_lookup_acquire(&apei_cd, 0)) == NULL) {
               error = ENXIO;
               goto out;
       }
       sc = device_private(apei0);

       /*
        * Fail if there's no EINJ.
        */
       if (sc->sc_tab.einj == NULL) {
               error = ENODEV;
               goto out;
       }

       /*
        * Perform the GET_ERROR_TYPE action and return the value to
        * sysctl.
        *
        * XXX Should this do it between BEGIN_INJECTION_OPERATION and
        * END_OPERATION?
        */
       types = apei_einj_act(sc, ACPI_EINJ_GET_ERROR_TYPE, 0);
       node.sysctl_data = &types;
       error = sysctl_lookup(SYSCTLFN_CALL(&node));

out:    if (apei0) {
               device_release(apei0);
               apei0 = NULL;
       }
       return error;
}