Saturday, March 23, 2013
Preloading Lua modules, part II
Well, four and a half megs per Lua state in the Kitchen Sink Lua interpreter. I thought about it, and I had Yet Another Idea™.
Lua not only has an array for preloaded modules, but an array of functions used to locate and load modules. So the idea I had was to insert two custom load functions—one to search for C based Lua modules, and one for Lua-based Lua modules. The code is pretty much straight forward:
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <lua.h>
#include <lauxlib.h>
#include <lualib.h>
/**************************************************************************/
typedef struct prelua_reg
{
const char *const name;
const char *const code;
const size_t *const size;
} prelua_reg__t;
/*************************************************************************/
int luaopen_org_conman_env (lua_State *);
int luaopen_org_conman_errno (lua_State *);
int luaopen_org_conman_fsys (lua_State *);
int luaopen_org_conman_math (lua_State *);
int luaopen_org_conman_syslog (lua_State *);
int luaopen_org_conman_hash (lua_State *);
int luaopen_org_conman_string_trim (lua_State *);
int luaopen_org_conman_string_wrap (lua_State *);
int luaopen_org_conman_string_remchar (lua_State *);
int luaopen_org_conman_process (lua_State *);
int luaopen_org_conman_net (lua_State *);
int luaopen_org_conman_dns (lua_State *);
int luaopen_org_conman_sys (lua_State *);
int luaopen_org_conman_uuid (lua_State *);
int luaopen_lpeg (lua_State *);
int luaopen_LuaXML_lib (lua_State *);
int luaopen_cURL (lua_State *);
/***********************************************************************/
extern const char c_org_conman_debug[];
extern const size_t c_org_conman_debug_size;
extern const char c_org_conman_getopt[];
extern const size_t c_org_conman_getopt_size;
extern const char c_org_conman_string[];
extern const size_t c_org_conman_string_size;
extern const char c_org_conman_table[];
extern const size_t c_org_conman_table_size;
extern const char c_org_conman_unix[];
extern const size_t c_org_conman_unix_size;
extern const char c_re[];
extern const size_t c_re_size;
extern const char c_LuaXml[];
extern const size_t c_LuaXml_size;
const luaL_Reg c_preload[] =
{
{ "LuaXML_lib" , luaopen_LuaXML_lib } ,
{ "cURL" , luaopen_cURL } ,
{ "lpeg" , luaopen_lpeg } ,
{ "org.conman.dns" , luaopen_org_conman_dns } ,
{ "org.conman.env" , luaopen_org_conman_env } ,
{ "org.conman.errno" , luaopen_org_conman_errno } ,
{ "org.conman.fsys" , luaopen_org_conman_fsys } ,
{ "org.conman.hash" , luaopen_org_conman_hash } ,
{ "org.conman.math" , luaopen_org_conman_math } ,
{ "org.conman.net" , luaopen_org_conman_net } ,
{ "org.conman.process" , luaopen_org_conman_process } ,
{ "org.conman.string.remchar" , luaopen_org_conman_string_remchar } ,
{ "org.conman.string.trim" , luaopen_org_conman_string_trim } ,
{ "org.conman.string.wrap" , luaopen_org_conman_string_wrap } ,
{ "org.conman.sys" , luaopen_org_conman_sys } ,
{ "org.conman.syslog" , luaopen_org_conman_syslog } ,
{ "org.conman.uuid" , luaopen_org_conman_uuid } ,
};
#define MAX_CMOD (sizeof(c_preload) / sizeof(luaL_Reg))
const prelua_reg__t c_luapreload[] =
{
{ "LuaXml" , c_LuaXml , &c_LuaXml_size } ,
{ "org.conman.debug" , c_org_conman_debug , &c_org_conman_debug_size } ,
{ "org.conman.getopt" , c_org_conman_getopt , &c_org_conman_getopt_size } ,
{ "org.conman.string" , c_org_conman_string , &c_org_conman_string_size } ,
{ "org.conman.table" , c_org_conman_table , &c_org_conman_table_size } ,
{ "org.conman.unix" , c_org_conman_unix , &c_org_conman_unix_size } ,
{ "re" , c_re , &c_re_size } ,
};
#define MAX_LMOD (sizeof(c_luapreload) / sizeof(prelua_reg__t))
/*************************************************************************/
static int luaLReg_cmp(const void *needle,const void *haystack)
{
const char *key = needle;
const luaL_Reg *value = haystack;
return (strcmp(key,value->name));
}
/*************************************************************************/
static int preloadlua_cloader(lua_State *const L)
{
const char *key;
const luaL_Reg *target;
key = luaL_checkstring(L,1);
target = bsearch(key,c_preload,MAX_CMOD,sizeof(luaL_Reg),luaLReg_cmp);
if (target == NULL)
lua_pushnil(L);
else
lua_pushcfunction(L,target->func);
return 1;
}
/************************************************************************/
static int preluareg_cmp(const void *needle,const void *haystack)
{
const char *key = needle;
const prelua_reg__t *value = haystack;
return (strcmp(key,value->name));
}
/*************************************************************************/
static int preloadlua_lualoader(lua_State *const L)
{
const char *key;
const prelua_reg__t *target;
key = luaL_checkstring(L,1);
target = bsearch(key,c_luapreload,MAX_LMOD,sizeof(prelua_reg__t),preluareg_cmp);
if (target == NULL)
lua_pushnil(L);
else
{
int rc = luaL_loadbuffer(L,target->code,*target->size,target->name);
if (rc != 0)
lua_pushnil(L);
}
return 1;
}
/***********************************************************************/
void preload_lua(lua_State *const L)
{
assert(L != NULL);
lua_gc(L,LUA_GCSTOP,0);
luaL_openlibs(L);
lua_gc(L,LUA_GCRESTART,0);
/*---------------------------------------------------------------
; modify the package.loaders[] array to include two new searchers:
;
; 1) scan for a C based module, return luaopen_*()
; 2) scan for a Lua based module, return the result of luaL_loadbuffer()
;---------------------------------------------------------------------*/
lua_getglobal(L,"package");
lua_getfield(L,-1,"loaders");
int len = lua_objlen(L,-1);
/*-----------------------------------------------------------------------
; insert the two new functions at the start of the package.loaders[]
; array---this way, we get first crack at loading the modules and don't
; waste time with expensive disk lookups.
;----------------------------------------------------------------------*/
for (int i = len + 2 ; i > 3 ; i--)
{
lua_rawgeti(L,-1,i - 2);
lua_rawseti(L,-2,i);
}
lua_pushinteger(L,1);
lua_pushcfunction(L,preloadlua_cloader);
lua_settable(L,-3);
lua_pushinteger(L,2);
lua_pushcfunction(L,preloadlua_lualoader);
lua_settable(L,-3);
}
And a quick test of the new Kitchen Sink Lua interpeter on this:
-- ensure any accumulated garbage is reclaimed
collectgarbage('collect')
collectgarbage('collect')
collectgarbage('collect')
print(collectgarbage('count') * 1024)
reveals a nice usage of 17,618 bytes—on par with the stock Lua interpreter. What's happening here is that the modules are no longer being shoved into the Lua state regardless of use (and there's one module that accounts for about 3½ megabytes—it's rarely used, but I do need it in some circumstances); they're now loaded into the Lua state as needed.
This also lead me to the concept of compressing the Lua written modules with zlib to save space in the executable (and it does help). I'll leave that code to the reader as an exercise.
Interestingly enough, the only hard part of this was trying to figure out
how to insert two elements at the start of an array using the C API of Lua—there is no
equivalent function to the Lua table.insert() function. I
resorted to checking the source code to table.insert() to see
how it was done.
The only other problem I had was debugging the zlib-based
version of this code—a typo (two missing characters—sigh) lead me on a
multi-hour bug chase.
But it works now, and I've decreased memory usage quite a bit with some few simple changes to the code, which is always nice.
