Wednesday, March 22, 2023
Preloading Lua modules, part III
I received an email from Andy Weidenbaum today, thanking me for writing a post about embedding Lua code into an executable which helped him in his project. On the plus side, it's nice that a post of mine was able to help him. On the non-plus side, I wrote that post ten years ago tomorrow!
Geeze, where does the time go?
I replied to him saying that I have since updated the code
(and sent him some copies)
but I think I should at least mention it here on the blog.
The first major change is populating the Lua array package.preload with the C-based Lua modules
(which is probably the intent of that array).
The second major change was compressing the Lua-based Lua modules using zlib to save space
(and the decompression time on modern systems is near negligible).
I accomplish this via a custom tool and the following rule in the makefile:
BIN2C = bin2c %.o : %.lua $(BIN2C) -9 -o $*.l.c -t $(NS)$(*F) $< $(CC) $(CFLAGS) -c -o $@ $*.l.c $(RM) $*.l.c
This defines an implicit rule to convert a .lua file into a .o (object) file that can be linked.
The first line will read the Lua source file,
compress it
(via the -9 option for heaviest compression)
and convert it to a C file.
The C file is then compiled into an object file,
and the C file is then removed as it's no longer needed.
The third major change is the routine now expects to initialize the entire Lua state, set up the embedded C and Lua based modules, then run the Lua “application” by name (I went from making a Kitchen Sink Lua interpreter to stand alone apps mainly written in Lua).
There are other minor changes, but I think at this point it's best to show some code:
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <zlib.h>
#include <lua.h>
#include <lauxlib.h>
#include <lualib.h>
#if LUA_VERSION_NUM == 501
# define lua_rawlen(L,idx) lua_objlen((L),(idx))
# define luaL_setfuncs(L,reg,up) luaI_openlib((L),NULL,(reg),(up))
# define SEARCHERS "loaders"
# define lua_load(L,f,d,k,x) (lua_load)((L),(f),(d),(k))
#else
# define SEARCHERS "searchers"
#endif
/**************************************************************************/
typedef struct prelua_reg
{
char const *const name;
unsigned char const *const code;
size_t const *const size;
} prelua_reg__s;
struct zlib_data
{
char const *name;
z_stream sin;
Bytef buffer[LUAL_BUFFERSIZE];
};
/**************************************************************************/
static char const *preloadlua_reader(lua_State *L,void *ud,size_t *size)
{
assert(L != NULL);
assert(ud != NULL);
assert(size != NULL);
struct zlib_data *data = ud;
(void)L;
data->sin.next_out = data->buffer;
data->sin.avail_out = sizeof(data->buffer);
inflate(&data->sin,Z_SYNC_FLUSH);
*size = sizeof(data->buffer) - data->sin.avail_out;
return (char const *)data->buffer;
}
/*************************************************************************/
static int preload_lualoader(lua_State *const L)
{
char const *key = luaL_checkstring(L,1);
prelua_reg__s const *target;
assert(L != NULL);
lua_getfield(L,LUA_REGISTRYINDEX,"org.conman:prelua");
lua_getfield(L,-1,key);
target = lua_touserdata(L,-1);
if (target == NULL)
{
lua_pushfstring(L,"\n\tno precompiled module '%s'",key);
return 1;
}
else
{
struct zlib_data data;
data.sin.zalloc = Z_NULL;
data.sin.zfree = Z_NULL;
data.sin.opaque = Z_NULL;
data.sin.next_in = (Byte *)target->code;
data.sin.avail_in = *target->size;
inflateInit(&data.sin);
lua_load(L,preloadlua_reader,&data,key,NULL);
inflateEnd(&data.sin);
lua_pushliteral(L,":preload:");
return 2;
}
}
/***********************************************************************/
int exec_lua_app(
lua_State *L,
char const *modname,
luaL_Reg const *preload,
prelua_reg__s const *prelua,
int argc,
char **argv
)
{
int preluasize;
assert(L != NULL);
assert(modname != NULL);
assert(preload != NULL);
assert(prelua != NULL);
assert(argc > 0);
assert(argv != NULL);
for (preluasize = 0 ; prelua[preluasize].name != NULL ; preluasize++)
;
lua_createtable(L,0,preluasize);
for (int i = 0 ; i < preluasize ; i++)
{
lua_pushlightuserdata(L,(void *)&prelua[i]);
lua_setfield(L,-2,prelua[i].name);
}
lua_setfield(L,LUA_REGISTRYINDEX,"org.conman:prelua");
luaL_openlibs(L);
lua_getglobal(L,"package");
lua_getfield(L,-1,"preload");
luaL_setfuncs(L,preload,0);
lua_getglobal(L,"package");
lua_getfield(L,-1,SEARCHERS);
for (lua_Integer i = lua_rawlen(L,-1) + 1 ; i > 1 ; i--)
{
lua_rawgeti(L,-1,i - 1);
lua_rawseti(L,-2,i);
}
lua_pushinteger(L,2);
lua_pushcfunction(L,preload_lualoader);
lua_settable(L,-3);
lua_pop(L,4);
lua_createtable(L,argc,0);
for (int i = 0 ; i < argc ; i++)
{
lua_pushinteger(L,i);
lua_pushstring(L,argv[i]);
lua_settable(L,-3);
}
lua_setglobal(L,"arg");
lua_pushcfunction(L,preload_lualoader);
lua_pushstring(L,modname);
lua_call(L,1,1);
return lua_pcall(L,0,0,0);
}
And to show how it's used, I'll use my gopher server as an example:
int main(int argc,char *argv[])
{
static luaL_Reg const c_preload[] =
{
{ "lpeg" , luaopen_lpeg } ,
{ "org.conman.clock" , luaopen_org_conman_clock } ,
{ "org.conman.errno" , luaopen_org_conman_errno } ,
{ "org.conman.fsys" , luaopen_org_conman_fsys } ,
{ "org.conman.fsys.magic" , luaopen_org_conman_fsys_magic } ,
{ "org.conman.math" , luaopen_org_conman_math } ,
{ "org.conman.net" , luaopen_org_conman_net } ,
{ "org.conman.pollset" , luaopen_org_conman_pollset } ,
{ "org.conman.process" , luaopen_org_conman_process } ,
{ "org.conman.signal" , luaopen_org_conman_signal } ,
{ "org.conman.syslog" , luaopen_org_conman_syslog } ,
{ "port70.getuserdir" , luaopen_port70_getuserdir } ,
{ "port70.setugid" , luaopen_port70_setugid } ,
{ NULL , NULL }
};
static prelua_reg__s const c_prelua[] =
{
{ "org.conman.const.exit" , c_org_conman_const_exit , &c_org_conman_const_exit_size } ,
{ "org.conman.const.gopher-types" , c_org_conman_const_gopher_types , &c_org_conman_const_gopher_types_size } ,
{ "org.conman.net.ios" , c_org_conman_net_ios , &c_org_conman_net_ios_size } ,
{ "org.conman.nfl" , c_org_conman_nfl , &c_org_conman_nfl_size } ,
{ "org.conman.nfl.tcp" , c_org_conman_nfl_tcp , &c_org_conman_nfl_tcp_size } ,
{ "org.conman.parsers.abnf" , c_org_conman_parsers_abnf , &c_org_conman_parsers_abnf_size } ,
{ "org.conman.parsers.ascii.char" , c_org_conman_parsers_ascii_char , &c_org_conman_parsers_ascii_char_size } ,
{ "org.conman.parsers.ascii.control" , c_org_conman_parsers_ascii_control , &c_org_conman_parsers_ascii_control_size } ,
{ "org.conman.parsers.ip-text" , c_org_conman_parsers_ip_text , &c_org_conman_parsers_ip_text_size } ,
{ "org.conman.parsers.iso.char" , c_org_conman_parsers_iso_char , &c_org_conman_parsers_iso_char_size } ,
{ "org.conman.parsers.iso.control" , c_org_conman_parsers_iso_control , &c_org_conman_parsers_iso_control_size } ,
{ "org.conman.parsers.mimetype" , c_org_conman_parsers_mimetype , &c_org_conman_parsers_mimetype_size } ,
{ "org.conman.parsers.url" , c_org_conman_parsers_url , &c_org_conman_parsers_url_size } ,
{ "org.conman.parsers.url.gopher" , c_org_conman_parsers_url_gopher , &c_org_conman_parsers_url_gopher_size } ,
{ "org.conman.parsers.utf8.char" , c_org_conman_parsers_utf8_char , &c_org_conman_parsers_utf8_char_size } ,
{ "org.conman.parsers.utf8.control" , c_org_conman_parsers_utf8_control , &c_org_conman_parsers_utf8_control_size } ,
{ "port70" , c_port70 , &c_port70_size } ,
{ "port70.cgi" , c_port70_cgi , &c_port70_cgi_size } ,
{ "port70.handlers.content" , c_port70_handlers_content , &c_port70_handlers_content_size } ,
{ "port70.handlers.file" , c_port70_handlers_file , &c_port70_handlers_file_size } ,
{ "port70.handlers.filesystem" , c_port70_handlers_filesystem , &c_port70_handlers_filesystem_size } ,
{ "port70.handlers.http" , c_port70_handlers_http , &c_port70_handlers_http_size } ,
{ "port70.handlers.sample" , c_port70_handlers_sample , &c_port70_handlers_sample_size } ,
{ "port70.handlers.url" , c_port70_handlers_url , &c_port70_handlers_url_size } ,
{ "port70.handlers.userdir" , c_port70_handlers_userdir , &c_port70_handlers_userdir_size } ,
{ "port70.mklink" , c_port70_mklink , &c_port70_mklink_size } ,
{ "port70.readfile" , c_port70_readfile , &c_port70_readfile_size } ,
{ "port70.safetext" , c_port70_safetext , &c_port70_safetext_size } ,
{ "re" , c_re , &c_re_size } ,
{ NULL , NULL , NULL }
};
lua_State *L = luaL_newstate();
if (L == NULL)
{
fprintf(stderr,"Cannot create Lua state\n");
return EXIT_FAILURE;
}
int rc = exec_lua_app(L,"port70",c_preload,c_prelua,argc,argv);
if (rc != LUA_OK)
fprintf(stderr,"lua_pcall() = %s\n",lua_tostring(L,-1));
lua_close(L);
return rc == LUA_OK ? EXIT_SUCCESS : EXIT_FAILURE;
}
I haven't yet written code to automagically write the main() routine,
but for a small application like port70 it wasn't that bad.
There are several approaches to automating this,
like overriding require() with some custom code to record loaded modules,
or building the list from LuaRocks,
but that's a project for another day.
