Search code examples
linuxluaprotocol-buffersaerospike

Aerospike Lua Protobuf pb.so: undefined symbol: lua_settop

I try to run simple lua udf in docker and keep getting this error:

ERROR lua create error: error loading module 'protobuf.pb' from file '/usr/local/lib/lua/5.1/protobuf/pb.so':
        /usr/local/lib/lua/5.1/protobuf/pb.so: undefined symbol: lua_settop

I searched over the whole interner and tried everything I was able to understand how to apply but nothing helped. I can successfully build an image, register my modules but when I run simple command it just fails with the error above:

Here is Dockerfile where I test it

FROM rockylinux:8.9
# install requirements
RUN dnf update -y  && \
    dnf install -y git gcc make findutils wget unzip readline-devel python3 python3-pip && \
    ln -sf /usr/bin/python3 /usr/bin/python
## install aerospile
RUN mkdir -p aerospike \
    && wget -qO- "https://download.aerospike.com/artifacts/aerospike-server-enterprise/6.3.0.5/aerospike-server-enterprise_6.3.0.5_tools-8.4.0_el8_$(uname -m).tgz" | tar -xvzf - -C ./aerospike --strip-components=1 \
    && cd ./aerospike && ./asinstall && cd ../ && rm -rf ./aerospike
# https://aerospike.com/developer/udf/knowing_lua#lua-version
ARG LUA_VERSION=5.1.4
# Use latest
ARG LUAROCKS_VERSION=3.11.1

# install lua
RUN mkdir -p lua && \
    wget -qO- https://www.lua.org/ftp/lua-${LUA_VERSION}.tar.gz | tar -xvzf - -C ./lua --strip-components=1 && \
    cd ./lua && CFLAGS="-fPIC -Wall -Wextra" LIBFLAGS="-shared" make -j $(nproc) linux && make -j $(nproc) install && \
    cd ../ && rm -rf ./lua && \
    lua -v && \
    # install luarocks
    mkdir -p ./luarocks && \
    wget -qO- luarocks https://luarocks.org/releases/luarocks-${LUAROCKS_VERSION}.tar.gz | tar -xvzf - -C ./luarocks --strip-components=1 && \
    cd luarocks && ./configure && make -j $(nproc) && make -j $(nproc) install && cd .. && rm -rf luarocks && \
    luarocks && \
    # install protobuf
    luarocks install protobuf  && \
    luarocks install lua-protobuf && \
    pip3 install protobuf && \
    dnf install -y protobuf-compiler && \
    # clean up
    dnf clean all

COPY ./proto /proto
COPY ./udf /udf

ENTRYPOINT ["asd"]
CMD ["--foreground"]

and the lua module

local protobuf = require 'protobuf'

function count(recs)
  return recs :
  map(function(rec)
    return 1
  end) :
  aggregate(0, function(count, rec)
    return count + 1
  end)
end

when I execute this function, it fails with the error above

aql> REGISTER MODULE '/udf/myudf.lua'
OK, 1 module added.

aql> AGGREGATE myudf.count() ON test
Solution

  • I prepared Dockerfile and instructions to demo it. It looks like there is a difference on how execute vs aggregate functions deal with imports. Please, notice in the examples below, that both errorudf and fineudf have local PartnerUserId = require "PartnerUserId_pb" which in its turn has local protobuf = require 'protobuf' , but only aggregate call is failing.

    It seems like an issue of Aerospike itself.

    > docker build -t as-lua .

> docker run -d --rm --name as-lua as-lua

> docker exec as-lua aql -c "register module '/fineudf.lua'"             
register module '/fineudf.lua'
OK, 1 module added.

> docker exec as-lua aql -c "register module '/errorudf.lua'"            
register module '/errorudf.lua'
OK, 1 module added.


> docker exec as-lua aql -c "execute fineudf.make() on test where PK='1'"
execute fineudf.make() on test where PK='1'
+------+
| make |
+------+
|      |
+------+
1 row in set (0.001 secs)

OK

> docker exec as-lua aql -c "aggregate errorudf.count() on test"            
2024-09-10 15:59:04 ERROR lua create error: error loading module 'protobuf.pb' from file '/usr/local/lib/lua/5.1/protobuf/pb.so':
        /usr/local/lib/lua/5.1/protobuf/pb.so: undefined symbol: lua_settop
Error: (100) UDF: Execution Error 1

aggregate errorudf.count() on test

    Dockerfile

    FROM rockylinux:8.9
# install requirements
RUN dnf update -y  && \
    dnf install -y git gcc-c++ make findutils wget unzip readline-devel python3 python3-pip nano && \
    ln -sf /usr/bin/python3 /usr/bin/python
## install aerospile
RUN mkdir -p aerospike \
    && wget -qO- "https://download.aerospike.com/artifacts/aerospike-server-enterprise/6.3.0.5/aerospike-server-enterprise_6.3.0.5_tools-8.4.0_el8_$(uname -m).tgz" | tar -xvzf - -C ./aerospike --strip-components=1 \
    && cd ./aerospike && ./asinstall && cd ../ && rm -rf ./aerospike
# https://aerospike.com/developer/udf/knowing_lua#lua-version
ENV LUA_VERSION=5.1.4
# Use latest
ENV LUAROCKS_VERSION=3.11.1
# install lua
RUN mkdir -p lua && \
    wget -qO- https://www.lua.org/ftp/lua-${LUA_VERSION}.tar.gz | tar -xvzf - -C ./lua --strip-components=1 && \
    cd ./lua && make -j $(nproc) linux && make -j $(nproc) linux install && \
    cd ../ && rm -rf ./lua && \
    lua -v && \
    # install luarocks
    mkdir -p ./luarocks && \
    wget -qO- luarocks https://luarocks.org/releases/luarocks-${LUAROCKS_VERSION}.tar.gz | tar -xvzf - -C ./luarocks --strip-components=1 && \
    cd luarocks && ./configure && make -j $(nproc) && make -j $(nproc) install && cd .. && rm -rf luarocks && \
    luarocks && \
    # install protobuf
    luarocks install protobuf  && \
    pip3 install protobuf && \
    dnf install -y protobuf-compiler && \
    # clean up
    dnf clean all
RUN cat >/PartnerUserId_pb.lua <<EOL
local module = {}
local protobuf = require 'protobuf'
module.PARTNERUSERID = protobuf.Descriptor()
module.PARTNERUSERID_USERID_FIELD = protobuf.FieldDescriptor()
module.PARTNERUSERID_TIMEOFMAPPING_FIELD = protobuf.FieldDescriptor()
module.PARTNERUSERID_USERID_FIELD.name = 'userId'
module.PARTNERUSERID_USERID_FIELD.full_name = '.PartnerUserId.userId'
module.PARTNERUSERID_USERID_FIELD.number = 1
module.PARTNERUSERID_USERID_FIELD.index = 0
module.PARTNERUSERID_USERID_FIELD.label = 2
module.PARTNERUSERID_USERID_FIELD.has_default_value = false
module.PARTNERUSERID_USERID_FIELD.default_value = ''
module.PARTNERUSERID_USERID_FIELD.type = 9
module.PARTNERUSERID_USERID_FIELD.cpp_type = 9
module.PARTNERUSERID_TIMEOFMAPPING_FIELD.name = 'timeOfMapping'
module.PARTNERUSERID_TIMEOFMAPPING_FIELD.full_name = '.PartnerUserId.timeOfMapping'
module.PARTNERUSERID_TIMEOFMAPPING_FIELD.number = 2
module.PARTNERUSERID_TIMEOFMAPPING_FIELD.index = 1
module.PARTNERUSERID_TIMEOFMAPPING_FIELD.label = 2
module.PARTNERUSERID_TIMEOFMAPPING_FIELD.has_default_value = false
module.PARTNERUSERID_TIMEOFMAPPING_FIELD.default_value = 0
module.PARTNERUSERID_TIMEOFMAPPING_FIELD.type = 3
module.PARTNERUSERID_TIMEOFMAPPING_FIELD.cpp_type = 2
module.PARTNERUSERID.name = 'PartnerUserId'
module.PARTNERUSERID.full_name = '.PartnerUserId'
module.PARTNERUSERID.nested_types = {}
module.PARTNERUSERID.enum_types = {}
module.PARTNERUSERID.fields = {module.PARTNERUSERID_USERID_FIELD, module.PARTNERUSERID_TIMEOFMAPPING_FIELD}
module.PARTNERUSERID.is_extendable = false
module.PARTNERUSERID.extensions = {}
module.PartnerUserId = protobuf.Message(module.PARTNERUSERID)
module.MESSAGE_TYPES = {'PartnerUserId'}
module.ENUM_TYPES = {}
return module
EOL
RUN cat >/fineudf.lua <<EOL
local PartnerUserId = require "PartnerUserId_pb"
function make(rec)
    local partner = PartnerUserId.PartnerUserId()
    partner.userId = '1'
    partner.timeOfMapping = os.time(os.date("!*t"))
    data = partner:SerializeToString()
    msg = PartnerUserId.PartnerUserId()
    return msg:ParseFromString(data)
end
EOL
RUN cat >/errorudf.lua <<EOL
local PartnerUserId = require "PartnerUserId_pb"
function count(recs)
    return recs:
    map(function(rec)
        return 1
    end)       :
    aggregate(0, function(count, rec)
        return count + 1
    end)
end
EOL
CMD ["asd", "--foreground"]