The Boston Diaries

The ongoing saga of a programmer who doesn't live in Boston, nor does he even like Boston, but yet named his weblog/journal “The Boston Diaries.”

Go figure.

Saturday, March 23, 2013

Preloading Lua modules, part II

Well, four and a half megs per Lua state in the Kitchen Sink Lua interpreter. I thought about it, and I had Yet Another Idea™.

Lua not only has an array for preloaded modules, but an array of functions used to locate and load modules. So the idea I had was to insert two custom load functions—one to search for C based Lua modules, and one for Lua-based Lua modules. The code is pretty much straight forward:

#include <stdlib.h>
#include <string.h>
#include <assert.h>

#include <lua.h>
#include <lauxlib.h>
#include <lualib.h>

/**************************************************************************/

typedef struct prelua_reg
{
  const char   *const name;
  const char   *const code;
  const size_t *const size;
} prelua_reg__t;

/*************************************************************************/

int	luaopen_org_conman_env		(lua_State *);
int	luaopen_org_conman_errno	(lua_State *);
int	luaopen_org_conman_fsys		(lua_State *);
int	luaopen_org_conman_math		(lua_State *);
int	luaopen_org_conman_syslog	(lua_State *);
int	luaopen_org_conman_hash		(lua_State *);
int	luaopen_org_conman_string_trim	(lua_State *);
int	luaopen_org_conman_string_wrap	(lua_State *);
int	luaopen_org_conman_string_remchar (lua_State *);
int	luaopen_org_conman_process	(lua_State *);
int	luaopen_org_conman_net		(lua_State *);
int	luaopen_org_conman_dns		(lua_State *);
int	luaopen_org_conman_sys		(lua_State *);
int	luaopen_org_conman_uuid		(lua_State *);
int	luaopen_lpeg			(lua_State *);
int	luaopen_LuaXML_lib		(lua_State *);
int	luaopen_cURL			(lua_State *);

/***********************************************************************/

extern const char   c_org_conman_debug[];
extern const size_t c_org_conman_debug_size;
extern const char   c_org_conman_getopt[];
extern const size_t c_org_conman_getopt_size;
extern const char   c_org_conman_string[];
extern const size_t c_org_conman_string_size;
extern const char   c_org_conman_table[];
extern const size_t c_org_conman_table_size;
extern const char   c_org_conman_unix[];
extern const size_t c_org_conman_unix_size;
extern const char   c_re[];
extern const size_t c_re_size;
extern const char   c_LuaXml[];
extern const size_t c_LuaXml_size;

const luaL_Reg c_preload[] =
{
  { "LuaXML_lib"		, luaopen_LuaXML_lib			} ,
  { "cURL"			, luaopen_cURL				} ,
  { "lpeg"			, luaopen_lpeg				} ,
  { "org.conman.dns"		, luaopen_org_conman_dns		} ,
  { "org.conman.env"		, luaopen_org_conman_env		} ,
  { "org.conman.errno"		, luaopen_org_conman_errno		} ,
  { "org.conman.fsys"		, luaopen_org_conman_fsys		} ,
  { "org.conman.hash"		, luaopen_org_conman_hash		} ,
  { "org.conman.math"		, luaopen_org_conman_math		} ,
  { "org.conman.net"		, luaopen_org_conman_net		} ,
  { "org.conman.process"	, luaopen_org_conman_process		} ,
  { "org.conman.string.remchar"	, luaopen_org_conman_string_remchar	} ,
  { "org.conman.string.trim"	, luaopen_org_conman_string_trim	} ,
  { "org.conman.string.wrap"	, luaopen_org_conman_string_wrap	} ,
  { "org.conman.sys"		, luaopen_org_conman_sys		} ,
  { "org.conman.syslog"		, luaopen_org_conman_syslog		} ,
  { "org.conman.uuid"		, luaopen_org_conman_uuid		} ,
};

#define MAX_CMOD (sizeof(c_preload) / sizeof(luaL_Reg))

const prelua_reg__t c_luapreload[] =
{
  { "LuaXml"			, c_LuaXml		, &c_LuaXml_size		} ,
  { "org.conman.debug"		, c_org_conman_debug	, &c_org_conman_debug_size	} ,
  { "org.conman.getopt"		, c_org_conman_getopt	, &c_org_conman_getopt_size	} ,
  { "org.conman.string"		, c_org_conman_string	, &c_org_conman_string_size	} ,
  { "org.conman.table"		, c_org_conman_table	, &c_org_conman_table_size	} ,
  { "org.conman.unix"		, c_org_conman_unix	, &c_org_conman_unix_size	} ,
  { "re"			, c_re			, &c_re_size			} ,
};

#define MAX_LMOD (sizeof(c_luapreload) / sizeof(prelua_reg__t))

/*************************************************************************/

static int luaLReg_cmp(const void *needle,const void *haystack)
{
  const char     *key   = needle;
  const luaL_Reg *value = haystack;
  
  return (strcmp(key,value->name));
}

/*************************************************************************/

static int preloadlua_cloader(lua_State *const L)
{
  const char     *key;
  const luaL_Reg *target;
  
  key    = luaL_checkstring(L,1);
  target = bsearch(key,c_preload,MAX_CMOD,sizeof(luaL_Reg),luaLReg_cmp);
  if (target == NULL)
    lua_pushnil(L);
  else
    lua_pushcfunction(L,target->func);
  return 1;
}

/************************************************************************/

static int preluareg_cmp(const void *needle,const void *haystack)
{
  const char          *key   = needle;
  const prelua_reg__t *value = haystack;
  
  return (strcmp(key,value->name));
}

/*************************************************************************/

static int preloadlua_lualoader(lua_State *const L)
{
  const char          *key;
  const prelua_reg__t *target;
  
  key = luaL_checkstring(L,1);
  target = bsearch(key,c_luapreload,MAX_LMOD,sizeof(prelua_reg__t),preluareg_cmp);
  if (target == NULL)
    lua_pushnil(L);
  else
  {
    int rc = luaL_loadbuffer(L,target->code,*target->size,target->name);
    if (rc != 0)
      lua_pushnil(L);
  }
  return 1;
}

/***********************************************************************/

void preload_lua(lua_State *const L)
{
  assert(L != NULL);
  
  lua_gc(L,LUA_GCSTOP,0);
  luaL_openlibs(L);
  lua_gc(L,LUA_GCRESTART,0);
  
  /*---------------------------------------------------------------
  ; modify the package.loaders[] array to include two new searchers:
  ;
  ; 1) scan for a C based module, return luaopen_*()
  ; 2) scan for a Lua based module, return the result of luaL_loadbuffer()
  ;---------------------------------------------------------------------*/
  
  lua_getglobal(L,"package");
  lua_getfield(L,-1,"loaders");
  
  int len = lua_objlen(L,-1);
  
  /*-----------------------------------------------------------------------
  ; insert the two new functions at the start of the package.loaders[]
  ; array---this way, we get first crack at loading the modules and don't
  ; waste time with expensive disk lookups.
  ;----------------------------------------------------------------------*/

  for (int i = len + 2 ; i > 3 ; i--)
  {
    lua_rawgeti(L,-1,i - 2);
    lua_rawseti(L,-2,i);
  }
  
  lua_pushinteger(L,1);
  lua_pushcfunction(L,preloadlua_cloader);
  lua_settable(L,-3);
  
  lua_pushinteger(L,2);
  lua_pushcfunction(L,preloadlua_lualoader);
  lua_settable(L,-3);    
}

And a quick test of the new Kitchen Sink Lua interpeter on this:

-- ensure any accumulated garbage is reclaimed
collectgarbage('collect')
collectgarbage('collect')
collectgarbage('collect')
print(collectgarbage('count') * 1024)

reveals a nice usage of 17,618 bytes—on par with the stock Lua interpreter. What's happening here is that the modules are no longer being shoved into the Lua state regardless of use (and there's one module that accounts for about 3½ megabytes—it's rarely used, but I do need it in some circumstances); they're now loaded into the Lua state as needed.

This also lead me to the concept of compressing the Lua written modules with zlib to save space in the executable (and it does help). I'll leave that code to the reader as an exercise.

Interestingly enough, the only hard part of this was trying to figure out how to insert two elements at the start of an array using the C API of Lua—there is no equivalent function to the Lua table.insert() function. I resorted to checking the source code to table.insert() to see how it was done.

The only other problem I had was debugging the zlib-based version of this code—a typo (two missing characters—sigh) lead me on a multi-hour bug chase.

But it works now, and I've decreased memory usage quite a bit with some few simple changes to the code, which is always nice.

Obligatory Picture

[It's the most wonderful time of the year!]

Obligatory Links

Obligatory Miscellaneous

You have my permission to link freely to any entry here. Go ahead, I won't bite. I promise.

The dates are the permanent links to that day's entries (or entry, if there is only one entry). The titles are the permanent links to that entry only. The format for the links are simple: Start with the base link for this site: http://boston.conman.org/, then add the date you are interested in, say 2000/08/01, so that would make the final URL:

http://boston.conman.org/2000/08/01

You can also specify the entire month by leaving off the day portion. You can even select an arbitrary portion of time.

You may also note subtle shading of the links and that's intentional: the “closer” the link is (relative to the page) the “brighter” it appears. It's an experiment in using color shading to denote the distance a link is from here. If you don't notice it, don't worry; it's not all that important.

It is assumed that every brand name, slogan, corporate name, symbol, design element, et cetera mentioned in these pages is a protected and/or trademarked entity, the sole property of its owner(s), and acknowledgement of this status is implied.

Copyright © 1999-2019 by Sean Conner. All Rights Reserved.