* * * * *

                            Preloading Lua modules

I'm tasked with testing the call processing on “Project: Wolowizard.” M
suggested, and I concurred, that using Lua [1] to manage the testing scripts
would be a Good Thing™. Easier to write and modify the tests as needed. So
over the past few years I've written a number of modules to handle the files
and protocols used in the project (one side effect: by re-implemeting the
code to read/write the various data files helped to verify the specification
and flush out architectural dependencies in the binary formats).

But one problem did exist: Not all the systems I need to run the test on have
Lua installed, and LuaRocks [2] has … um … “issues” on our Solaris boxes
(otherwise, it's not that bad a package manager [3]). So I decided to build
what I call “Kitchen Sink Lua”—a Lua interpreter that has the 47 modules
required to run the testing scripts (okay, eight of the modules are already
built into Lua).

It took some time to wrangle, as some of the modules were written in Lua (so
the source needed to be embedded) and I had to figure out how to integrate
some third party modules (like LuaCURL [4]) into the build system, but
perhaps the hardest bit was to ensure the modules were initialized properly.
My first attempt, while it worked (mostly by accident) wasn't technically
correct (as I realized when I read this message on a mailing list [5]).

I then restructured my code, which not only made it correct, but smaller and
clearer.

> #include <stdlib.h>
> #include <assert.h>
>
> #include <lua.h>
> #include <lauxlib.h>
> #include <lualib.h>
>
> /**************************************************************************/
>
> typedef struct prelua_reg
> {
>   const char   *const name;
>   const char   *const code;
>   const size_t *const size;
> } prelua_reg__t;
>
> /*************************************************************************/
>
> int   luaopen_org_conman_env          (lua_State *);
> int   luaopen_org_conman_errno        (lua_State *);
> int   luaopen_org_conman_fsys         (lua_State *);
> int   luaopen_org_conman_math         (lua_State *);
> int   luaopen_org_conman_syslog       (lua_State *);
> int   luaopen_org_conman_hash         (lua_State *);
> int   luaopen_org_conman_string_trim  (lua_State *);
> int   luaopen_org_conman_string_wrap  (lua_State *);
> int   luaopen_org_conman_string_remchar (lua_State *);
> int   luaopen_org_conman_process      (lua_State *);
> int   luaopen_org_conman_net          (lua_State *);
> int   luaopen_org_conman_dns          (lua_State *);
> int   luaopen_org_conman_sys          (lua_State *);
> int   luaopen_org_conman_uuid         (lua_State *);
> int   luaopen_lpeg                    (lua_State *);
> int   luaopen_LuaXML_lib              (lua_State *);
> int   luaopen_cURL                    (lua_State *);
>
> /***********************************************************************/
>
>       /*---------------------------------------------------------------
>       ; Modules written in Lua.  The build system takes the Lua code,
>       ; processes it through luac (the Lua compiler), then creates an
>       ; object file which exports a character array containing the byte
>       ; code, and a variable which gives the size of the bytecode array.
>       ;---------------------------------------------------------------*/
>
> extern const char   c_org_conman_debug[];
> extern const size_t c_org_conman_debug_size;
> extern const char   c_org_conman_getopt[];
> extern const size_t c_org_conman_getopt_size;
> extern const char   c_org_conman_string[];
> extern const size_t c_org_conman_string_size;
> extern const char   c_org_conman_table[];
> extern const size_t c_org_conman_table_size;
> extern const char   c_org_conman_unix[];
> extern const size_t c_org_conman_unix_size;
> extern const char   c_re[];
> extern const size_t c_re_size;
> extern const char   c_LuaXml[];
> extern const size_t c_LuaXml_size;
>
>       /*----------------------------------------------------------------
>       ; Modules written in C.  We can use luaL_register() to load these
>       ; into package.preloaded[]
>       ;----------------------------------------------------------------*/
>
> const luaL_Reg c_preload[] =
> {
>   { "org.conman.env"          , luaopen_org_conman_env                } ,
>   { "org.conman.errno"                , luaopen_org_conman_errno              } ,
>   { "org.conman.fsys"         , luaopen_org_conman_fsys               } ,
>   { "org.conman.math"         , luaopen_org_conman_math               } ,
>   { "org.conman.syslog"               , luaopen_org_conman_syslog             } ,
>   { "org.conman.hash"         , luaopen_org_conman_hash               } ,
>   { "org.conman.string.trim"  , luaopen_org_conman_string_trim        } ,
>   { "org.conman.string.wrap"  , luaopen_org_conman_string_wrap        } ,
>   { "org.conman.string.remchar"       , luaopen_org_conman_string_remchar     } ,
>   { "org.conman.process"      , luaopen_org_conman_process            } ,
>   { "org.conman.net"          , luaopen_org_conman_net                } ,
>   { "org.conman.dns"          , luaopen_org_conman_dns                } ,
>   { "org.conman.sys"          , luaopen_org_conman_sys                } ,
>   { "org.conman.uuid"         , luaopen_org_conman_uuid               } ,
>   { "lpeg"                    , luaopen_lpeg                          } ,
>   { "LuaXML_lib"              , luaopen_LuaXML_lib                    } ,
>   { "cURL"                    , luaopen_cURL                          } ,
>   { NULL                      , NULL                                  }
> };
>
>       /*---------------------------------------------------------------
>       ; Modules written in Lua.  These need to be loaded and populated
>       ; into package.preloaded[] by some code provided in this file.
>       ;----------------------------------------------------------------
>
> const prelua_reg__t c_luapreload[] =
> {
>   { "org.conman.debug"                , c_org_conman_debug    , &c_org_conman_debug_size      } ,
>   { "org.conman.getopt"               , c_org_conman_getopt   , &c_org_conman_getopt_size     } ,
>   { "org.conman.string"               , c_org_conman_string   , &c_org_conman_string_size     } ,
>   { "org.conman.table"                , c_org_conman_table    , &c_org_conman_table_size      } ,
>   { "org.conman.unix"         , c_org_conman_unix     , &c_org_conman_unix_size       } ,
>   { "re"                      , c_re                  , &c_re_size                    } ,
>   { "LuaXml"                  , c_LuaXml              , &c_LuaXml_size                } ,
>   { NULL                      , NULL                  , NULL                          }
> };
>
> /*************************************************************************/
>
> void preload_lua(lua_State *const L)
> {
>   assert(L != NULL);
>
>   lua_gc(L,LUA_GCSTOP,0);
>   luaL_openlibs(L);
>   lua_gc(L,LUA_GCRESTART,0);
>
>   /*---------------------------------------------------------------
>   ; preload all the modules.  This does does not initialize them,
>   ; just makes them available for require().
>   ;
>   ; I'm doing it this way because of a recent email on the LuaJIT
>   ; email list:
>   ;
>   ; http://www.freelists.org/post/luajit/Trivial-bug-in-bitops-bitc-luaopen-bit,4
>   ;
>   ; Pre-loading these modules in package.preload[] means that they're be
>   ; initialized properly through the require() statement.
>   ;---------------------------------------------------------------------*/
>
>   lua_getglobal(L,"package");
>   lua_getfield(L,-1,"preload");
>
>   luaL_register(L,NULL,c_preload);
>   for (size_t i = 0 ; c_luapreload[i].name != NULL ; i++)
>   {
>     int rc = luaL_loadbuffer(L,c_luapreload[i].code,*c_luapreload[i].size,c_luapreload[i].name);
>     if (rc != 0)
>     {
>       const char *err;
>
>       switch(rc)
>       {
>         case LUA_ERRRUN:    err = "runtime error"; break;
>         case LUA_ERRSYNTAX: err = "syntax error";  break;
>         case LUA_ERRMEM:    err = "memory error";  break;
>         case LUA_ERRERR:    err = "generic error"; break;
>         case LUA_ERRFILE:   err = "file error";    break;
>         default:            err = "unknown error"; break;
>       }
>
>       fprintf(stderr,"%s: %s\n",c_luapreload[i].name,err);
>       exit(EXIT_FAILURE);
>     }
>     lua_setfield(L,-2,c_luapreload[i].name);
>   }
> }
>
> /*************************************************************************/
>

Yes, this is the code used in “Project: Wolowizard” (minus the proprietary
modules) and is a good example of the module preload feature in Lua. The
modules in C are easy to build (the following is from the Makefile):

> obj/spc/process.o : $(LUASPC)/src/process.c     \
>                 $(LUA)/lua.h                    \
>                 $(LUA)/lauxlib.h
>         $(CC) $(CFLAGS) -I$(LUA) -c -o $@ $<
>

While the Lua-based modules are a bit more involved:

> obj/spc/unix.o : $(LUASPC)/lua/unix.lua $(BIN2C) $(LUAC)
>         $(LUAC) -o tmp/unix.out $<
>         $(BIN2C) -o tmp/unix.c -t org_conman_unix tmp/unix.out
>         $(CC) $(CFLAGS) -c -o $@ tmp/unix.c
>

These modules are compiled using luac (which outputs the Lua byte code used
by the core Lua VM (Virtual Machine)), then through a program that converts
this output into a C file, which is then compiled into an object file that
can be linked into the final Kitchen Sink Lua interpreter.

[1] http://www.lua.org/
[2] http://www.luarocks.org/
[3] gopher://gopher.conman.org/0Phlog:2013/02/25.2
[4] http://luacurl.luaforge.net/
[5] http://www.freelists.org/post/luajit/Trivial-bug-in-bitops-bitc-luaopen-bit,4

Email author at [email protected]

You are viewing proxied material from gopher.conman.org. The copyright of proxied material belongs to its original authors. Any comments or complaints in relation to proxied material should be directed to the original authors of the content concerned. Please see the disclaimer for more details.